From ab81139da4ad3b70c1b268d200aa86af223669d9 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 00:28:25 -0800 Subject: [PATCH 1/7] [None][feat] Attach feature and sparse metadata state as extensions AttentionMetadata becomes a small core plus pluggable MetadataExtension objects. Features, sparse algorithms and models attach their per-step state to a core metadata family instead of subclassing it. Callers reach it with metadata.ext(T) / metadata.find_ext(T), and the composed metadata class forwards each extension's declared fields. Framework (attention/backends/extensions.py): - MetadataExtension declares its fields, shared state, creation config and draft-copy plan; the core drives allocate/prepare and the CUDA-graph, draft and attention-DP hooks. - ExtensionRegistry resolves requirement ordering; CUDA-graph and copy.copy views get their own clones. - AttentionBackend.metadata_extensions and create_metadata(); core families attach their own via default_extension_types(). - KV-length corrections (kv_lens_offsets) are applied inside prepare(), so one prepare call produces complete, idempotent state. Features (attention/backends/features): HelixMetadata, DraftMetadata, FlashMlaMetadata, Fp4MlaState, SpecDecodingMetadata, CrossAttentionMetadata, and RecurrentMetadata in place of the core mamba fields. Sparse algorithms are SparseAlgorithmExtension subclasses on TrtllmAttentionMetadata instead of metadata subclasses: RocketMetadata, QSAMetadata, DSAMetadata, DeepseekV4Metadata, CSA2Metadata, MiniMaxM3TritonMetadata, MiniMaxM3MsaMetadata. MTPIndexShareExtension replaces the MTPIndexShareMetadata protocol. Consumers (FMHA libraries, MLA, models, modeling_v2 targets, speculative workers, executor) and tests read the extensions; the attention developer guide describes the extension contract. Signed-off-by: Yihan Wang --- scripts/generate_fmha_params.py | 14 +- .../_experimental/modeling_v2/README.md | 16 +- .../modeling.py | 97 +- .../glm_5_2_nvfp4__sm_103__tp8/attention.py | 25 +- .../cross_rank_topk_global_emit/bridge.py | 16 +- .../kernels/decode_prologue/__init__.py | 7 +- .../kernels/decode_prologue/prologue.py | 417 +-- .../kernels/draft_loop/draft_loop.py | 10 +- .../kernels/draft_loop/draft_step.py | 353 +- .../kernels/draft_loop/verify.py | 74 +- .../kernels/rope_kv_append/ops.py | 18 +- .../glm_5_2_nvfp4__sm_103__tp8/runtime.py | 6 + .../glm_5_2_nvfp4__sm_103__tp8/spec_worker.py | 154 +- .../_dsa_backend.py | 18 +- .../_indexer.py | 50 +- .../_mtp_worker.py | 67 +- .../spec_worker.py | 412 +-- .../spec_worker.py | 412 +-- .../minimax_m3_nvfp4__sm_103__tp4/modeling.py | 98 +- .../minimax_m3_nvfp4__sm_103__tp8/modeling.py | 98 +- .../minimax_m3_nvfp4__sm_107__tp2/modeling.py | 113 +- .../ops/spec/m3_spec_offsets.py | 60 - .../minimax_m3_nvfp4__sm_107__tp4/modeling.py | 116 +- .../ops/spec/m3_spec_offsets.py | 60 - .../minimax_m3_nvfp4__sm_107__tp8/modeling.py | 113 +- .../ops/spec/m3_spec_offsets.py | 60 - .../modeling.py | 17 +- .../attention/ATTENTION_DEVELOPER_GUIDE.md | 201 +- .../_torch/attention/backends/core.py | 802 +++++ .../_torch/attention/backends/extensions.py | 695 ++++ .../attention/backends/features/__init__.py | 18 + .../attention/backends/features/cross.py | 247 ++ .../attention/backends/features/draft.py | 246 ++ .../attention/backends/features/flash_mla.py | 141 + .../attention/backends/features/helix.py | 282 ++ .../attention/backends/features/recurrent.py | 75 + .../backends/features/spec_decoding.py | 422 +++ .../_torch/attention/backends/flashinfer.py | 3205 +++++++++-------- .../attention/backends/fmha/cute_dsl_mla.py | 31 +- .../_torch/attention/backends/fmha/fp4_mla.py | 24 +- .../attention/backends/fmha/msa_prefill.py | 6 +- .../_torch/attention/backends/fmha/phased.py | 18 +- .../backends/fp4_mla/cache_gather.py | 8 +- .../backends/fp4_mla/cache_update.py | 68 +- .../attention/backends/fp4_mla/decode.py | 39 +- .../backends/fp4_mla/fp4_mla_context.py | 37 +- .../attention/backends/fp4_mla/layout.py | 9 +- .../attention/backends/fp4_mla/metadata.py | 117 +- .../attention/backends/fp4_mla/packed_plan.py | 10 +- .../attention/backends/fp4_mla/state.py | 144 +- .../attention/backends/fp4_mla/v_cache.py | 7 +- .../_torch/attention/backends/interface.py | 1222 +++---- .../attention/backends/sparse/csa2/backend.py | 8 +- .../backends/sparse/csa2/cache_manager.py | 49 +- .../backends/sparse/csa2/decoder_replay.py | 93 +- .../attention/backends/sparse/csa2/indexer.py | 2 +- .../attention/backends/sparse/csa2/kernel.py | 62 +- .../backends/sparse/csa2/metadata.py | 1070 ++++-- .../backends/sparse/deepseek_v4/__init__.py | 3 +- .../sparse/deepseek_v4/cache_manager.py | 6 + .../backends/sparse/deepseek_v4/metadata.py | 583 +-- .../attention/backends/sparse/dsa/__init__.py | 3 +- .../attention/backends/sparse/dsa/indexer.py | 54 +- .../attention/backends/sparse/dsa/metadata.py | 1917 +++++----- .../attention/backends/sparse/dsa/module.py | 3 - .../sparse/minimax_m3/cache_manager.py | 2 +- .../sparse/minimax_m3/kernels/msa_utils.py | 2 +- .../backends/sparse/minimax_m3/msa_backend.py | 386 +- .../sparse/minimax_m3/triton_metadata.py | 115 +- .../attention/backends/sparse/offload.py | 12 +- .../attention/backends/sparse/params.py | 124 +- .../attention/backends/sparse/qsa/metadata.py | 422 +-- .../backends/sparse/rocket/__init__.py | 3 +- .../backends/sparse/rocket/metadata.py | 415 +-- .../_torch/attention/backends/trtllm.py | 2333 ++++++------ .../_torch/attention/backends/vanilla.py | 38 +- .../_torch/models/modeling_deepseekv4.py | 11 +- .../_torch/models/modeling_deepseekv41.py | 19 +- .../models/modeling_glm5_next_vision.py | 4 +- .../_torch/models/modeling_minimaxm3.py | 49 +- .../_torch/models/modeling_qwen3_next.py | 17 +- .../_torch/models/modeling_qwen3vl.py | 2 +- .../_torch/models/modeling_qwen4_exp.py | 35 +- .../_torch/models/modeling_speculative.py | 5 - .../_torch/pyexecutor/cuda_graph_runner.py | 136 +- .../_torch/pyexecutor/engine/metadata.py | 60 +- .../engine/runners/decoder/graphs.py | 42 +- .../engine/runners/decoder/inputs.py | 392 +- .../engine/runners/decoder/metadata.py | 82 +- .../engine/runners/decoder/runner.py | 83 +- .../engine/runners/decoder/warmup.py | 42 +- .../pyexecutor/engine/runners/encoder.py | 23 +- .../engine/runners/encoder_decoder.py | 58 +- .../kv_cache/kv_cache_manager_v2.py | 8 + .../_torch/pyexecutor/resource_manager.py | 9 + .../_torch/pyexecutor/runtime_hooks.py | 17 +- .../_torch/pyexecutor/steady_spec_prepare.py | 20 +- tensorrt_llm/_torch/speculative/__init__.py | 4 - tensorrt_llm/_torch/speculative/dflash.py | 445 +-- .../_torch/speculative/draft_target.py | 273 +- tensorrt_llm/_torch/speculative/eagle3.py | 239 +- .../_torch/speculative/eagle3_dynamic_tree.py | 71 +- tensorrt_llm/_torch/speculative/interface.py | 185 +- tensorrt_llm/_torch/speculative/mtp.py | 222 +- .../_torch/speculative/mtp_dynamic_tree.py | 230 +- tensorrt_llm/_torch/speculative/pard.py | 303 +- .../tools/layer_wise_benchmarks/runner.py | 38 +- .../attention_perf/attention_perf_harness.py | 5 +- tests/microbenchmarks/bench_fp4_mla_decode.py | 9 +- .../test_metadata_extension_contract.py | 812 +++++ .../test_ragged_metadata_lifecycle.py | 70 +- .../attention/multi_gpu/helix_test_utils.py | 4 +- .../multi_gpu/test_mla_helix_context.py | 5 + .../_torch/attention/sparse/csa2/_utils.py | 110 +- .../attention/sparse/csa2/test_backend.py | 88 +- .../sparse/csa2/test_cache_manager.py | 82 +- .../sparse/csa2/test_context_only_metadata.py | 6 +- .../attention/sparse/csa2/test_indexer.py | 57 +- .../sparse/csa2/test_indexer_descriptors.py | 92 +- .../attention/sparse/csa2/test_kernel.py | 69 +- .../attention/sparse/csa2/test_metadata.py | 400 +- .../sparse/csa2/test_metadata_uploads.py | 25 +- .../attention/sparse/csa2/test_module.py | 45 +- .../deepseek_v4/test_compressor_module.py | 32 +- .../deepseek_v4/test_deepseek_v4_offload.py | 70 +- .../test_deepseek_v4_sparse_mla.py | 41 +- .../deepseek_v4/test_indexer_slot_mapping.py | 24 +- .../test_ragged_fused_q_rope_specs.py | 33 +- .../deepseek_v4/test_token_major_rows.py | 143 +- .../attention/sparse/dsa/test_dsa_indexer.py | 513 ++- .../sparse/dsa/test_dsa_sparse_mla.py | 26 +- .../sparse/dsa/test_metadata_topk_init.py | 52 +- .../sparse/dsa/test_mtp_schedule_skip.py | 53 +- .../sparse/dsa/test_req_idx_per_token.py | 69 +- .../msa/test_minimax_m3_sparse_attn_decode.py | 23 +- .../attention/sparse/msa/test_msa_backend.py | 145 +- .../sparse/msa/test_msa_runtime_policy.py | 64 +- .../qsa/test_qsa_cache_manager_layout.py | 23 + .../attention/sparse/qsa/test_qsa_sparse.py | 291 +- .../sparse/rocketkv/test_rocketkv.py | 75 +- .../attention/sparse/test_sparse_attention.py | 7 +- .../attention/sparse/test_sparse_mha.py | 23 +- .../sparse/test_sparse_mla_forward.py | 4 +- .../attention/sparse/test_sparse_mqa_gqa.py | 24 +- .../test_attention_failure_logging.py | 6 +- .../_torch/attention/test_attention_mla.py | 70 +- .../_torch/attention/test_cute_dsl_mla.py | 10 +- .../attention/test_cute_dsl_mla_split_plan.py | 10 +- .../attention/test_draft_metadata_reuse.py | 780 ++++ .../attention/test_flashinfer_attention.py | 129 +- .../test_flashinfer_decode_tensor_cores.py | 4 +- .../_torch/attention/test_fmha_interface.py | 34 +- .../unittest/_torch/attention/test_fp4_mla.py | 147 +- .../_torch/attention/test_fp4_mla_packed.py | 29 +- .../_torch/attention/test_fp4_mla_splitkv.py | 22 +- .../attention/test_helix_context_slots.py | 14 +- .../test_metadata_creation_config.py | 176 + .../test_mixed_precision_backends.py | 310 +- .../test_pixtral_head104_attention.py | 20 +- .../test_trtllm_paged_context_fmha_gate.py | 9 +- tests/unittest/_torch/helpers.py | 121 + .../modeling/_deepseek_v41_test_utils.py | 3 +- .../_torch/modeling/test_deepseek_v41_ced.py | 15 +- .../test_deepseek_v41_multimodal_backbone.py | 2 +- .../modeling/test_modeling_deepseekv4.py | 6 +- .../modeling/test_modeling_deepseekv41.py | 78 +- .../_torch/modeling/test_modeling_gemma3.py | 9 +- .../_torch/modeling/test_modeling_gemma4.py | 46 +- .../_torch/modeling/test_modeling_llama.py | 117 +- .../modeling/test_qsa_runtime_wiring.py | 15 +- .../_torch/modeling/test_qwen4_exp_support.py | 3 + .../spec/test_modeling_v2_mtp_step.py | 54 +- .../spec/_draft_step_op_matrix.py | 13 +- .../spec/test_modeling_v2_small_batch_mtp.py | 32 +- .../models/kimi_k3_vl/_kimi_k3_decode_step.py | 17 +- .../test_k3_decode_worker.py | 25 +- .../attention/_m3_attn_runtime_tp4.py | 2 +- .../test_modeling_v2_m3_dispatch.py | 5 +- .../attention/_m3_attn_runtime_tp8.py | 2 +- .../test_modeling_v2_m3_dispatch.py | 5 +- .../attention/_m3_attn_runtime_tp2.py | 2 +- ...est_modeling_v2_m3_block_table_contract.py | 2 +- .../test_modeling_v2_m3_n_valid_contract.py | 3 +- .../spec/test_modeling_v2_m3_spec_offsets.py | 139 - .../attention/_m3_attn_runtime_tp4.py | 2 +- .../test_modeling_v2_m3_n_valid_contract.py | 3 +- .../spec/test_modeling_v2_m3_spec_offsets.py | 139 - .../attention/_m3_attn_runtime_tp8.py | 2 +- .../test_modeling_v2_m3_n_valid_contract.py | 3 +- .../spec/test_modeling_v2_m3_spec_offsets.py | 139 - .../unittest/_torch/models/test_minimax_m3.py | 20 +- .../mamba/test_flashinfer_chunk_gdn.py | 9 +- .../test_deepseek_v41_attention_dp.py | 47 +- .../multi_gpu/test_minimax_m3_piecewise.py | 8 + .../engine/runners/decoder/test_inputs.py | 2 +- .../engine/runners/decoder/test_metadata.py | 14 +- .../engine/runners/decoder/test_runner.py | 101 - .../engine/runners/decoder/test_warmup.py | 20 +- .../engine/runners/decoder/utils.py | 3 +- .../pyexecutor/engine/runners/test_encoder.py | 18 +- .../engine/test_decoder_runtime_hooks.py | 10 +- .../_torch/pyexecutor/engine/test_metadata.py | 62 +- .../_torch/pyexecutor/engine_test_utils.py | 15 +- .../kv_cache/test_kvcm2_integration.py | 18 +- .../pyexecutor/test_context_logits_memory.py | 1 - .../test_cuda_graph_capture_replay.py | 180 +- .../pyexecutor/test_cuda_graph_runner.py | 21 - .../pyexecutor/test_kv_lens_update_once.py | 105 +- .../test_preprocess_inputs_kv_lens_hook.py | 69 - .../_torch/pyexecutor/test_runtime_hooks.py | 18 +- .../pyexecutor/test_steady_spec_prepare.py | 48 +- .../test_dflash_ctx_arena_sizing.py | 6 +- .../test_dflash_ctx_block_tables.py | 4 +- .../test_dflash_flashinfer_metadata.py | 35 - .../test_dflash_guided_rollback.py | 6 +- .../hw_agnostic/test_dflash_worker.py | 6 +- .../test_dspark_model_engine_runtime.py | 90 +- .../hw_agnostic/test_pard_guided_rollback.py | 9 +- .../_torch/speculative/test_eagle3.py | 113 +- .../speculative/test_force_accepted_tokens.py | 34 +- .../speculative/test_spec_tail_kernels.py | 7 +- .../test_trtllm_attention_metadata.py | 8 +- tests/unittest/utils/metadata_contract.py | 74 + 223 files changed, 17254 insertions(+), 12119 deletions(-) delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/ops/spec/m3_spec_offsets.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/ops/spec/m3_spec_offsets.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/ops/spec/m3_spec_offsets.py create mode 100644 tensorrt_llm/_torch/attention/backends/core.py create mode 100644 tensorrt_llm/_torch/attention/backends/extensions.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/__init__.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/cross.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/draft.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/flash_mla.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/helix.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/recurrent.py create mode 100644 tensorrt_llm/_torch/attention/backends/features/spec_decoding.py create mode 100644 tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py create mode 100644 tests/unittest/_torch/attention/test_draft_metadata_reuse.py create mode 100644 tests/unittest/_torch/attention/test_metadata_creation_config.py delete mode 100644 tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/spec/test_modeling_v2_m3_spec_offsets.py delete mode 100644 tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/spec/test_modeling_v2_m3_spec_offsets.py delete mode 100644 tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/spec/test_modeling_v2_m3_spec_offsets.py delete mode 100644 tests/unittest/_torch/pyexecutor/test_preprocess_inputs_kv_lens_hook.py delete mode 100644 tests/unittest/_torch/speculative/hw_agnostic/test_dflash_flashinfer_metadata.py create mode 100644 tests/unittest/utils/metadata_contract.py diff --git a/scripts/generate_fmha_params.py b/scripts/generate_fmha_params.py index bbff4a496073..40284a6e7d00 100644 --- a/scripts/generate_fmha_params.py +++ b/scripts/generate_fmha_params.py @@ -65,6 +65,13 @@ "RopeParams": "AttentionRopeParams", } +# Mirrored runtime classes whose members another class declares: the attention +# metadata keeps its state on its core and extensions, so a declaration-only +# class lists the members the native op reads, with their types. +MIRROR_DECLARATIONS = { + "TrtllmAttentionMetadata": "TrtllmAttentionMetadataNativeFields", +} + # Root fields whose C++ usage decides which mirrored members are emitted. USAGE_PREFIXES = ("attn", "meta") @@ -367,9 +374,10 @@ def _read_mirror( struct_name = _struct_name(class_name) if struct_name in mirrors: return mirrors[struct_name] - if class_name not in classes: - raise ValueError(f"mirrored class {class_name} not found in the schema sources") - members = _collect_members(classes[class_name], classes) + declaring_class = MIRROR_DECLARATIONS.get(class_name, class_name) + if declaring_class not in classes: + raise ValueError(f"mirrored class {declaring_class} not found in the schema sources") + members = _collect_members(classes[declaring_class], classes) if used is not None: unknown = sorted({path[0] for path in used} - members.keys()) if unknown: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 5a2ed0fb329b..b927aef8ce3d 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -318,15 +318,13 @@ mode the target uses. reserves per sequence, and leave the rewind of rejected tokens to the engine. Update `kv_lens` per draft step: step 0 drops the rejected drafts, and each later step adds one. -* **Attention metadata.** Restore the attention metadata fields the stock - MTP-Eagle worker restores after its draft loop: the sequence lengths, the - speculative packed mask, position offsets and generation lengths, and - `all_rank_num_tokens`. CUDA graph warmup replays the forward on the same - metadata. Save the lengths with `prepare_for_spec_dec` and restore them with - `restore_from_spec_dec` in a `finally` block, so a failed step, such as an - out-of-memory error during warmup, does not leave them changed. Like the - worker, the draft chain leaves `num_contexts`, `host_request_types` and - `kv_lens_cuda` changed. +* **Attention metadata.** Run the draft chain on the draft metadata, inside + `with attn_metadata.draft_loop() as draft:`, as the stock MTP-Eagle worker + does. The draft starts from the target's state on every replay, including + CUDA graph warmup, and the target's metadata is not changed. Change the + draft's lengths through its core methods (`set_seq_lens`, + `as_generation_rows`, `adjust_kv_lens`) and call + `attn_metadata.refresh_draft()` before the next draft forward. * **Per-token state.** Commit recurrent, conv or indexer state written per token for the accepted tokens only, for example through the cache manager's `update_mamba_states`. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py index 58e1cb306c0a..c0afeb2d3eaa 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py @@ -39,6 +39,8 @@ from torch import nn from transformers import PretrainedConfig +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import set_use_spec_decoding from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionInputType, @@ -2127,8 +2129,8 @@ class Glm5NextMtpStep: 3. Draft ``K = runtime_draft_len`` tokens: the MTP layer on (accepted token, target hidden state) rows, its shared head on the vocabulary shard, and the global argmax across the TP group; step 0 runs on the verify layout, steps 1.. on one row per request. - 4. Restore the attention metadata the chain rewrote (CUDA-graph warmup replays this forward on the same - metadata) and return what the speculative sampler reads. + 4. Return what the speculative sampler reads. The chain runs on the draft metadata (``draft_loop``), which + starts from the target's state on every replay, so the target's metadata is left as it was. Every shape is fixed by host integers of the step (batch size, number of contexts, K), so the step captures in one CUDA graph with the target. @@ -2137,7 +2139,6 @@ class Glm5NextMtpStep: def __init__(self, mapping: Mapping, vocab_size: int) -> None: self.mapping = mapping self.vocab_size = vocab_size - self._saved: dict[str, Any] = {} def __call__( self, @@ -2183,9 +2184,7 @@ def __call__( state_indices=attn_metadata.mamba_metadata.state_indices, ) - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - self._save_metadata(attn_metadata, batch_size) - try: + with attn_metadata.draft_loop() as draft_metadata: next_draft_tokens = self._draft( input_ids, position_ids, @@ -2193,19 +2192,15 @@ def __call__( accepted_tokens, num_accepted_tokens, attn_metadata, + draft_metadata, spec_metadata, draft_model, ) - finally: - self._restore_metadata(attn_metadata) - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens last_accepted = accepted_tokens[ spec_metadata.batch_indices_cuda[:batch_size], num_accepted_tokens - 1 ] next_new_tokens = torch.cat([last_accepted.unsqueeze(1), next_draft_tokens], dim=1) - attn_metadata.use_spec_decoding = True return { "logits": logits, "new_tokens": accepted_tokens, @@ -2260,11 +2255,13 @@ def _draft( hidden_states: torch.Tensor, accepted_tokens: torch.Tensor, num_accepted_tokens: torch.Tensor, + target_metadata: AttentionMetadata, attn_metadata: AttentionMetadata, spec_metadata: Any, draft_model: Glm5NextDraftModel, ) -> torch.Tensor: - """``next_draft_tokens [B, K]``. Rewrites the attention metadata's lengths as it goes; the caller restores.""" + """``next_draft_tokens [B, K]``. ``attn_metadata`` is ``target_metadata``'s draft metadata in its draft loop; + the chain advances its lengths as it goes.""" batch_size = attn_metadata.num_seqs num_contexts = attn_metadata.num_contexts num_gens = batch_size - num_contexts @@ -2319,7 +2316,13 @@ def _draft( step_positions[rows] if step_positions.dim() == 1 else step_positions[..., rows] ) + 1 self._advance_lengths( - attn_metadata, step, num_contexts, batch_size, draft_len, num_accepted_tokens + target_metadata, + attn_metadata, + step, + num_contexts, + batch_size, + draft_len, + num_accepted_tokens, ) return torch.stack(drafts, dim=1) @@ -2366,6 +2369,7 @@ def _draft_argmax(self, logits: torch.Tensor) -> torch.Tensor: @staticmethod def _advance_lengths( + target_metadata: AttentionMetadata, attn_metadata: AttentionMetadata, step: int, num_contexts: int, @@ -2376,62 +2380,25 @@ def _advance_lengths( """Lengths for the next draft step. After step 0 every request feeds one row: a generation request's KV length drops its rejected drafts and a context request's grows by its first draft, then each step adds one.""" if step == 0: - attn_metadata._seq_lens[:batch_size].fill_(1) - attn_metadata._seq_lens_cuda[:batch_size].fill_(1) - attn_metadata.on_update() + attn_metadata.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) has_kv_cache = attn_metadata.kv_cache_manager is not None - if has_kv_cache and hasattr(attn_metadata, "host_request_types"): - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) if has_kv_cache: - attn_metadata.num_contexts = 0 - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - draft_len - num_accepted_tokens[num_contexts:] + # The context requests continue as one-token generation rows. + attn_metadata.as_generation_rows() + if num_contexts > 0 and attn_metadata.find_ext(FlashMlaMetadata) is not None: + attn_metadata.block_ids_per_seq[:batch_size].copy_( + attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True + ) + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - draft_len, ) - attn_metadata.kv_lens_cuda[:num_contexts] += 1 - if ( - has_kv_cache - and num_contexts > 0 - and getattr(attn_metadata, "enable_flash_mla", False) - ): - attn_metadata.block_ids_per_seq[:batch_size].copy_( - attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True - ) - attn_metadata.update_for_spec_dec() - attn_metadata.use_spec_decoding = False + attn_metadata.adjust_kv_lens(slice(0, num_contexts), 1) + target_metadata.refresh_draft() + set_use_spec_decoding(attn_metadata, False) else: - attn_metadata.kv_lens_cuda[:batch_size] += 1 - attn_metadata.update_for_spec_dec() - - # -- attention metadata save / restore ---------------------------------- - - def _save_metadata(self, attn_metadata: AttentionMetadata, batch_size: int) -> None: - """Save what the draft chain overwrites: the sequence lengths (``prepare_for_spec_dec``) and the target's - speculative masks, position offsets and generation lengths.""" - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - saved = {} - if attn_metadata.spec_decoding_packed_mask is not None: - saved["packed_mask"] = attn_metadata.spec_decoding_packed_mask[:batch_size].clone() - if attn_metadata.spec_decoding_position_offsets is not None: - saved["position_offsets"] = attn_metadata.spec_decoding_position_offsets.clone() - if attn_metadata.spec_decoding_generation_lengths is not None: - saved["generation_lengths"] = attn_metadata.spec_decoding_generation_lengths[ - :batch_size - ].clone() - self._saved = saved - - def _restore_metadata(self, attn_metadata: AttentionMetadata) -> None: - if attn_metadata.has_spec_dec_saved_state: - attn_metadata.restore_from_spec_dec() - attn_metadata.on_update() - saved, self._saved = self._saved, {} - if "packed_mask" in saved: - mask = saved["packed_mask"] - attn_metadata.spec_decoding_packed_mask[: mask.shape[0]].copy_(mask) - if "position_offsets" in saved: - attn_metadata.spec_decoding_position_offsets.copy_(saved["position_offsets"]) - if "generation_lengths" in saved: - lengths = saved["generation_lengths"] - attn_metadata.spec_decoding_generation_lengths[: lengths.shape[0]].copy_(lengths) + attn_metadata.adjust_kv_lens(slice(0, batch_size), 1) + target_metadata.refresh_draft() # --------------------------------------------------------------------------- diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py index a01b42b429e6..f07e1292f35b 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py @@ -30,7 +30,10 @@ ) from tensorrt_llm._torch.attention.backends.sparse.dsa.backend import DSATrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.dsa.indexer import Indexer -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( + DSAMetadata, + DSAtrtllmAttentionMetadata, +) from tensorrt_llm._torch.attention.backends.sparse.dsa.module import DSAHooks from tensorrt_llm._torch.distributed import AllReduceParams from tensorrt_llm._torch.models.modeling_deepseekv3 import DeepseekV32Attention @@ -40,6 +43,11 @@ from .kernels.indexer_topk import region +def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: + """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" + return metadata.ext(DSAMetadata) + + def reclass(module, builtin: type, cls: type) -> None: """Re-class ``module``, which must be exactly the built-in class ``builtin``, as ``cls``. @@ -309,6 +317,7 @@ def sparse_attn_predict( forward_args: AttentionForwardArgs, ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: """Run the DSA indexer and convert its local TopK to paged indices.""" + dsa_state = _dsa_state(metadata) if dispatch.phase() is not dispatch.Phase.DECODE: return super().sparse_attn_predict( q=q, k=k, metadata=metadata, forward_args=forward_args @@ -360,7 +369,7 @@ def sparse_attn_predict( local_layer_idx = self.get_local_layer_idx(metadata) kv_cache_dtype = getattr(metadata.kv_cache_manager, "dtype", None) if kv_cache_dtype == tensorrt_llm.bindings.DataType.NVFP4 and not is_generation: - metadata._ensure_pool_view_cached() + dsa_state._ensure_pool_view_cached() page_index_scale, layer_offset = ( metadata.kv_cache_manager.get_primary_pool_page_index_params(local_layer_idx) ) @@ -390,19 +399,19 @@ def sparse_attn_predict( metadata.host_kv_cache_pool_pointers, metadata.host_kv_cache_pool_mapping, topk_indices, - metadata._cached_req_idx_ctx, - metadata._cached_block_table_ctx, + dsa_state._cached_req_idx_ctx, + dsa_state._cached_block_table_ctx, metadata.ctx_kv_indptr[: metadata.num_contexts + 1], scratch, compact_indices, self._nvfp4_mla_kv_scale_quant_orig, local_layer_idx, total_kv_tokens, - metadata._cached_tokens_per_block, - page_index_scale * metadata._cached_tokens_per_block, + dsa_state._cached_tokens_per_block, + page_index_scale * dsa_state._cached_tokens_per_block, layer_offset, metadata.kv_cache_manager.mla_kv_cache_residual_dim, - metadata._cached_num_pool_tokens, + dsa_state._cached_num_pool_tokens, ) metadata.nvfp4_mla_context_fp8_scratch = scratch forward_args.sparse_runtime_params.aux_kv_cache_pool_ptr = scratch.data_ptr() @@ -437,7 +446,7 @@ def sparse_attn_predict( self._nvfp4_mla_kv_scale_quant_orig, local_layer_idx, metadata.kv_cache_manager.mla_kv_cache_residual_dim, - metadata._cached_num_pool_tokens, + dsa_state._cached_num_pool_tokens, ) # Static sparse MLA treats indices as offsets from kvPtr. Feed it # compact offsets and the FP8 scratch base; TRTLLMGen is unchanged. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py index 88afc55c7912..45f36fe628c2 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py @@ -55,6 +55,16 @@ ] +def _dsa_state(metadata): + """The DSA extension of ``metadata``, which holds the grouped-remap state. + + Imported on use, as the rest of this module's attention imports. + """ + from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata + + return metadata.ext(DSAMetadata) + + class GlobalEmitContractError(RuntimeError): """The bridge work cross_rank_topk_global_emit performed at the top-k differs from what the call sites requested.""" @@ -65,7 +75,7 @@ class RemapPlan: ``grouped``: the leader of an active full + shared group remaps the whole group (``layer_ids``, ``page_index_scale``) and publishes it in - ``metadata._group_remap_batched[leader]``; the layer's own rows are + ``_dsa_state(metadata)._group_remap_batched[leader]``; the layer's own rows are ``out[slot]``. Otherwise the layer remaps alone (``page_index_scale``, ``layer_offset``). """ @@ -113,7 +123,7 @@ def predict_remap(metadata, local_layer: int) -> Optional[RemapPlan]: from tensorrt_llm._torch.attention.backends.sparse.dsa import backend as dsa_backend if dsa_backend._GROUP_REMAP and not metadata.in_mtp_draft_loop: - struct = metadata._ensure_group_remap_struct() + struct = _dsa_state(metadata)._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is not None and local_layer < len(leader_of): leader = leader_of[local_layer] @@ -281,6 +291,6 @@ def remap_topk_to_global(backend, topk: torch.Tensor, metadata, packet) -> torch packet.plan = None _check_plan(plan, backend, topk, metadata, packet) if plan.remap.grouped: - metadata._group_remap_batched[plan.remap.leader] = plan.pool + _dsa_state(metadata)._group_remap_batched[plan.remap.leader] = plan.pool return plan.pool[plan.remap.slot] return plan.pool diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/__init__.py index e4d3a07e65cd..39b5c691335d 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/__init__.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/__init__.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""decode_prologue: the target decode-metadata prologue and the target embedding in one launch. +"""decode_prologue: the target decode prologue and the target embedding in one launch. -decode_prologue extends draft_loop (``draft_loop``): its launch reuses draft_loop's DeepGEMM -schedule reproduction (``draft_loop/draft_step.py``) and draft_loop's embedding semantics -(``draft_loop/embedding.py``) for the target step instead of the draft steps. +decode_prologue extends draft_loop (``draft_loop``): its launch reuses draft_loop's embedding +semantics (``draft_loop/embedding.py``) for the target step instead of the draft steps. Entry point: :class:`prologue.DecodePrologueHook`, which ``runtime.make_runtime_hooks`` declares to the engine as the step loop's ``decode_prologue`` and hands to the decode diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/prologue.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/prologue.py index df7d601fd44b..a468d7194b62 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/prologue.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/decode_prologue/prologue.py @@ -12,34 +12,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Single-launch target decode-metadata prologue and target embedding (decode_prologue). - -A decode step's captured forward begins with ``DecoderInputPreparer.preprocess_inputs``: -``on_update_kv_lens()``, the overlap scheduler's accepted-count correction of -the position ids and the KV lengths, and ``on_update_kv_lens()`` again. On the -fused DSA decode-metadata path each call writes the 2-D KV-length view, three -DeepGEMM paged-MQA-logits schedules (each computed into a fresh tensor and -copied into its persistent buffer), the expanded lengths of the DSL atom split, -the fused decode metadata (request index, indexer K-cache slot mappings, -generation indptrs) and, when the indexer is skipped, the dense top-k rows. -The target model then embeds the input ids with the replicated ``Embedding`` -(a range mask, ``F.embedding`` and ``masked_fill_``). About thirty serial -launches precede the first layer. - -``target_prologue`` produces the identical device state with one launch: -program 0 applies the correction and derives every metadata buffer from the -corrected lengths held in registers (the state the second native call leaves; -the first call's writes are all overwritten by the second); programs 1.. copy -the embedding rows (``weight[id]``, or +0.0 for an id outside the table). The -host replays the host-side statements of both native calls. The schedule is -draft_loop's reproduction of DeepGEMM's SM100 partition, confirmed against the native -op by draft_loop's ``schedule_self_check``. Every decision is made on the host before -any device work; outside the supported domain the native code runs unchanged. - -The embedding rows are handed to the model forward through -``take_embedding``: the launch records ``(input_ids, weight, output)`` on the -step's attention metadata, and the model forward consumes it once for exactly -that ``input_ids`` tensor and embedding weight (otherwise it embeds natively). +"""Single-launch target decode prologue and target embedding (decode_prologue). + +A decode step's captured forward begins with ``DecoderInputPreparer.preprocess_inputs``, +which applies the overlap scheduler's accepted-count correction to the position ids; the +KV-length correction and the DSA decode metadata derived from the corrected lengths are +already in place, from the metadata's ``prepare(BatchInputs(kv_lens_offsets=...))``. +The target model then embeds the input ids with the replicated ``Embedding`` (a range +mask, ``F.embedding`` and ``masked_fill_``). + +``target_prologue`` produces the identical device state with one launch: program 0 +applies the position correction; programs 1.. copy the embedding rows (``weight[id]``, or ++0.0 for an id outside the table). Every decision is made on the host before any device work; +outside the supported domain the native code runs unchanged. + +The embedding rows are handed to the model forward through ``take_embedding``: the +launch records ``(attention metadata, input_ids, weight, output)`` on the model's +``DecodePrologueHook``, and the model forward consumes it once for exactly that +metadata, ``input_ids`` tensor and embedding weight (otherwise it embeds natively). """ from typing import Optional @@ -48,172 +38,40 @@ import triton import triton.language as tl -from ..draft_loop import draft_step as _draft_step from ..draft_loop import embedding as _draft_embedding -from ..draft_loop.draft_step import _write_schedule _MAX_BATCH = 64 _NUM_WARPS = 4 _BLOCK_H = 2048 -# Elements of one dense top-k tile (BLOCK_N rows x BLOCK_K columns). -_DENSE_TILE = 4096 -_PACKET_ATTR = "_decode_prologue_embedding" @triton.jit def _target_prologue_kernel( - kv_lens_ptr, # int32 [>= B] (in/out) - kv_off_ptr, # int32 [>= B] previous_kv_lens_offsets (CORRECT) - seq_lens_ptr, # int32 [>= B] pos_ptr, # int32 [>= N_POS] position ids of the step (in/out, CORRECT) pos_off_ptr, # int32 [>= N_POS] previous_pos_id_offsets (CORRECT) - kv2d_ptr, # int32 [max_seqs, cap] (out) - kv2d_stride, - sched_ptr, # int32 [(NUM_SMS + 1) * 2] (out) - sched_full_ptr, # int32 [(NUM_SMS + 1) * 2] (out) - kv_exp_ptr, # int32 [>= B * EXPAND] (out, EXPAND > 1) - sched_exp_ptr, # int32 [(NUM_SMS + 1) * 2] (out, EXPAND > 1) - block_offsets_ptr, # int32 [max_seqs, max_blocks] - block_offsets_stride_0, - req_idx_ptr, # int32 [>= num_tokens] (out) - slot_fp8_ptr, # int64 [>= num_tokens] (out) - slot_scale_ptr, # int64 [>= num_tokens] (out) - gen_kv_indptr_ptr, # int64 [>= B + 1] (out) - gen_cached_indptr_ptr, # int64 [>= B + 1] (out) - topk_ptr, # int32 [>= num_tokens, topk_stride] (out, DENSE) - topk_stride, ids_ptr, # int32/int64 [N_EMB] (EMBED) weight_ptr, # [vocab_size, HIDDEN] (EMBED) emb_ptr, # [N_EMB, HIDDEN] (out, EMBED) vocab_size, weight_row_stride, emb_row_stride, - num_tokens, N_POS: tl.constexpr, - B: tl.constexpr, CORRECT: tl.constexpr, - WIDTH: tl.constexpr, - EXPAND: tl.constexpr, - NUM_SMS: tl.constexpr, - SPLIT_KV: tl.constexpr, - max_blocks: tl.constexpr, - block_offsets_stride_1: tl.constexpr, - tokens_per_block: tl.constexpr, - data_bytes_per_token: tl.constexpr, - scale_size: tl.constexpr, - block_stride: tl.constexpr, - scale_base_offset: tl.constexpr, - DENSE: tl.constexpr, - TOPK: tl.constexpr, EMBED: tl.constexpr, HIDDEN: tl.constexpr, NCB: tl.constexpr, - BLOCK_B: tl.constexpr, - BLOCK_W: tl.constexpr, - BLOCK_E: tl.constexpr, - BLOCK_T: tl.constexpr, - BLOCK_SM: tl.constexpr, BLOCK_P: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, BLOCK_H: tl.constexpr, ): pid = tl.program_id(0) if pid == 0: - offs_b = tl.arange(0, BLOCK_B) - mask_b = offs_b < B - - # ----- accepted-count correction (in place) and the step's lengths --- - kv = tl.load(kv_lens_ptr + offs_b, mask=mask_b, other=0) + # ----- accepted-count correction of the position ids (in place) ----- if CORRECT: - kv = kv + tl.load(kv_off_ptr + offs_b, mask=mask_b, other=0) - tl.store(kv_lens_ptr + offs_b, kv, mask=mask_b) offs_p = tl.arange(0, BLOCK_P) mask_p = offs_p < N_POS pos = tl.load(pos_ptr + offs_p, mask=mask_p, other=0) pos = pos + tl.load(pos_off_ptr + offs_p, mask=mask_p, other=0) tl.store(pos_ptr + offs_p, pos, mask=mask_p) - kv = tl.where(mask_b, kv, 0) - seq = tl.load(seq_lens_ptr + offs_b, mask=mask_b, other=0) - seq = tl.where(mask_b, seq, 0) - - # ----- 2-D KV lengths and the paged-MQA-logits schedules ------------- - offs_w = tl.arange(0, BLOCK_W) - kv2d_mask = mask_b[:, None] & (offs_w[None, :] < WIDTH) - tl.store( - kv2d_ptr + offs_b[:, None] * kv2d_stride + offs_w[None, :], - kv[:, None] + tl.zeros((BLOCK_B, BLOCK_W), tl.int32), - mask=kv2d_mask, - ) - _write_schedule(kv, mask_b, B, sched_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) - _write_schedule(kv, mask_b, B, sched_full_ptr, WIDTH, NUM_SMS, SPLIT_KV, BLOCK_SM) - if EXPAND > 1: - offs_e = tl.arange(0, BLOCK_E) - mask_e = offs_e < B * EXPAND - owner = (offs_e[:, None] // EXPAND) == offs_b[None, :] - kv_e = tl.sum(tl.where(owner, kv[None, :], 0), 1) - tl.store(kv_exp_ptr + offs_e, kv_e, mask=mask_e) - _write_schedule(kv_e, mask_e, B * EXPAND, sched_exp_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) - - # ----- fused DSA decode metadata (the native kernel's arithmetic) ---- - kv64 = kv.to(tl.int64) - seq64 = seq.to(tl.int64) - tl.store(gen_kv_indptr_ptr, tl.full((), 0, tl.int64)) - tl.store(gen_cached_indptr_ptr, tl.full((), 0, tl.int64)) - tl.store(gen_kv_indptr_ptr + 1 + offs_b, tl.cumsum(kv64, 0), mask=mask_b) - tl.store(gen_cached_indptr_ptr + 1 + offs_b, tl.cumsum(kv64 - seq64, 0), mask=mask_b) - - flat_start = tl.cumsum(seq64, 0) - seq64 - start_pos = kv64 - seq64 - offs_t = tl.arange(0, BLOCK_T).to(tl.int64) - g = flat_start[:, None] + offs_t[None, :] - mask_t = mask_b[:, None] & (offs_t[None, :] < seq64[:, None]) & (g < num_tokens) - tl.store( - req_idx_ptr + g, offs_b[:, None] + tl.zeros((BLOCK_B, BLOCK_T), tl.int32), mask=mask_t - ) - gpos = start_pos[:, None] + offs_t[None, :] - blk = gpos // tokens_per_block - pos_in_blk = gpos % tokens_per_block - pos_in_blk = tl.where(pos_in_blk < 0, pos_in_blk + tokens_per_block, pos_in_blk) - blk = tl.minimum(tl.maximum(blk, 0), max_blocks - 1) - bid = tl.load( - block_offsets_ptr - + offs_b[:, None].to(tl.int64) * block_offsets_stride_0 - + blk * block_offsets_stride_1, - mask=mask_t, - other=0, - ).to(tl.int64) - tl.store( - slot_fp8_ptr + g, bid * block_stride + pos_in_blk * data_bytes_per_token, mask=mask_t - ) - tl.store( - slot_scale_ptr + g, - bid * block_stride + scale_base_offset + pos_in_blk * scale_size, - mask=mask_t, - ) - - # ----- dense top-k rows of a skipped indexer ------------------------ - if DENSE: - # position(g) = (kv - seq)[b] + (g - flat_start[b]) for the request b - # whose token range holds g (``_get_dense_topk_indices``). - offs_n = tl.arange(0, BLOCK_N).to(tl.int64) - seq_end = tl.cumsum(seq64, 0) - owned = (seq_end[None, :] <= offs_n[:, None]) & mask_b[None, :] - req = tl.sum(owned.to(tl.int32), 1) - onehot = req[:, None] == offs_b[None, :] - shift = tl.sum(tl.where(onehot, (start_pos - flat_start)[None, :], 0), 1) - tok_pos = offs_n + shift - mask_n = offs_n < num_tokens - for k0 in tl.static_range(0, TOPK, BLOCK_K): - offs_k = k0 + tl.arange(0, BLOCK_K) - rows = tl.where( - offs_k[None, :].to(tl.int64) <= tok_pos[:, None], offs_k[None, :], -1 - ) - tl.store( - topk_ptr + offs_n[:, None] * topk_stride + offs_k[None, :], - rows.to(tl.int32), - mask=mask_n[:, None] & (offs_k[None, :] < TOPK), - ) else: if EMBED: e = pid - 1 @@ -232,79 +90,6 @@ def _target_prologue_kernel( tl.store(emb_ptr + row.to(tl.int64) * emb_row_stride + cols, values, mask=col_mask) -def _metadata_ineligibility(md, B: int) -> Optional[str]: - """Why the fused-metadata part cannot replace the native calls, or None.""" - from tensorrt_llm._torch.attention.backends.sparse.dsa import is_dsa_cache_manager - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, - _fused_dsa_meta_enabled, - ) - - if not isinstance(md, DSAtrtllmAttentionMetadata): - return "metadata is not DSAtrtllmAttentionMetadata" - cls = type(md) - for name in ("on_update_kv_lens", "_compute_kv_lens_row_reorder", "get_indexer_kv_lens"): - if getattr(cls, name) is not getattr(DSAtrtllmAttentionMetadata, name): - return f"metadata overrides {name}" - if md.kv_cache_manager is None or not is_dsa_cache_manager(md.kv_cache_manager): - return "KV cache manager is not a DSA cache manager" - if not _fused_dsa_meta_enabled(): - return "native fused DSA metadata disabled" - if md.num_contexts != 0 or md.num_generations <= 0 or md.num_tokens <= 0: - return "not a generation-only step" - if md.use_fp8_ds_mla or getattr(md, "fp4_mla_state", None) is not None: - return "FP8-DS/FP4 MLA metadata state" - if getattr(md, "enable_helix", False): - return "helix metadata" - if md.enable_ragged_verification or md.is_ragged_verify: - return "ragged verification" - if md._indexer_compress_ratio > 1: - return "compressed indexer lengths" - if md.use_expanded_buffers_for_mtp: - return "expanded MTP buffers" - if md.max_draft_tokens <= 0: - return "no draft tokens" - if B <= 0 or B > _MAX_BATCH or B != md.num_seqs or B != md.num_generations: - return f"batch size {B} outside the fused domain" - if md.needs_gvr_prior and B * md.gen_token_stride >= 2 * md.num_sms: - return "GVR row reorder active" - host_seq = md.seq_lens[:B] - if int(host_seq.sum()) != md.num_tokens or int(host_seq.max()) > 1 + md.max_draft_tokens: - return "query lengths outside the decode layout" - int32_attrs = ( - "kv_lens_cuda", - "seq_lens_cuda", - "kv_lens_cuda_2d", - "indexer_k_cache_block_offsets", - "scheduler_metadata_buffer", - "scheduler_metadata_buffer_full_next_n", - "req_idx_per_token", - ) - for name in int32_attrs: - if getattr(md, name).dtype != torch.int32: - return f"unexpected dtype of {name}" - for name in ( - "slot_mapping_fp8", - "slot_mapping_scale", - "gen_kv_indptr", - "gen_cached_token_indptr", - ): - if getattr(md, name).dtype != torch.int64: - return f"unexpected dtype of {name}" - if md.expand_for_dsl and md.dsl_expand_factor > 1: - if md.kv_lens_expanded_cuda.dtype != torch.int32: - return "unexpected dtype of kv_lens_expanded_cuda" - if B * md.dsl_expand_factor > md.kv_lens_expanded_cuda.shape[0]: - return "expanded lengths exceed their buffer" - if md.skip_indexer_for_gen_reqs: - topk = md.topk_indices_buffer - if topk.dtype != torch.int32 or topk.stride(1) != 1 or topk.shape[1] != md.num_sparse_topk: - return "unexpected dense top-k buffer layout" - if _draft_step.schedule_self_check(md.num_sms, md.kv_lens_cuda.device) is None: - return "DeepGEMM schedule not reproduced" - return None - - def _correction_ineligibility(preparer, inputs, md, B: int) -> Optional[str]: """Why the accepted-count correction cannot be fused, or None.""" pos = inputs.get("position_ids") @@ -321,14 +106,13 @@ def _correction_ineligibility(preparer, inputs, md, B: int) -> Optional[str]: return "position ids share storage with the input ids" buffers = getattr(preparer, "_buffers", None) pos_off = getattr(buffers, "previous_pos_id_offsets_cuda", None) - kv_off = getattr(buffers, "previous_kv_lens_offsets_cuda", None) - if pos_off is None or kv_off is None: + if pos_off is None: return "no overlap-scheduler offsets" - if pos_off.dtype != torch.int32 or kv_off.dtype != torch.int32: + if pos_off.dtype != torch.int32: return "unexpected offset dtype" - if not pos_off.is_contiguous() or not kv_off.is_contiguous(): + if not pos_off.is_contiguous(): return "offsets are not contiguous" - if pos_off.shape[0] < num_pos or kv_off.shape[0] < B: + if pos_off.shape[0] < num_pos: return "offsets shorter than the step" if getattr(md, "num_chunked_ctx_requests", 0) != 0: return "chunked context rows" @@ -355,30 +139,6 @@ def _embedding_target(preparer, inputs): return embed, None -def _replay_host_on_update_kv_lens(md) -> None: - """Host statements of one fused-eligible ``on_update_kv_lens()`` (no device work).""" - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, - ) - from tensorrt_llm.logger import logger - - super(DSAtrtllmAttentionMetadata, md).on_update_kv_lens() - md._invalidate_pool_view_cache() - md.gen_indexer_kv_lens_cuda_runtime = md.get_indexer_kv_lens( - md.kv_lens_cuda[md.num_contexts : md.num_seqs] - ) - if not getattr(md, "_fused_dsa_meta_armed", False): - logger.info( - "Fused DSA decode-metadata " - f"kernel armed (num_seqs={md.num_seqs}, " - f"num_tokens={md.num_tokens})." - ) - md._fused_dsa_meta_armed = True - # In the eligible domain the row reorder is off, so this only resets the - # host attribute; the dense top-k rows are written by the launch. - md._compute_kv_lens_row_reorder() - - def prologue_ineligibility(preparer, inputs, enable_spec_decode: bool) -> Optional[str]: """Return why ``target_prologue`` must leave the native code in place, or None.""" md = inputs.get("attn_metadata") @@ -388,9 +148,10 @@ def prologue_ineligibility(preparer, inputs, enable_spec_decode: bool) -> Option if mapping is not None and mapping.has_cp_helix(): return "helix context parallelism" B = md.num_seqs - reason = _metadata_ineligibility(md, B) - if reason is not None: - return reason + if md.num_contexts != 0 or md.num_generations <= 0 or md.num_tokens <= 0: + return "not a generation-only step" + if B <= 0 or B > _MAX_BATCH: + return f"batch size {B} outside the fused domain" config = getattr(preparer, "_config", None) if config is None: return "no preparer config" @@ -399,12 +160,18 @@ def prologue_ineligibility(preparer, inputs, enable_spec_decode: bool) -> Option return None -def target_prologue(preparer, inputs, *, enable_spec_decode: bool, runtime_draft_len: int) -> bool: +def target_prologue( + preparer, + inputs, + *, + enable_spec_decode: bool, + runtime_draft_len: int, + hook: Optional["DecodePrologueHook"] = None, +) -> bool: """Run the decode-metadata prologue (and the target embedding) in one launch. - Replaces the device work and reproduces the host statements of - ``DecoderInputPreparer.preprocess_inputs`` up to (not including) the guided - decoder event, which the caller keeps. + Replaces the device work of ``DecoderInputPreparer.preprocess_inputs`` up to + (not including) the guided decoder event, which the caller keeps. Args: preparer: the ``DecoderInputPreparer`` (reads ``_config``, ``mapping``, @@ -431,34 +198,22 @@ def target_prologue(preparer, inputs, *, enable_spec_decode: bool, runtime_draft return False md = inputs["attn_metadata"] - B = md.num_seqs correct = enable_spec_decode and not preparer._config.disable_overlap_scheduler embed, embed_reason = _embedding_target(preparer, inputs) key = "target_embedding" if embed is not None else f"target_embedding[{embed_reason}]" dispatch.log_capability("decode_prologue", key, embed is not None, embed_reason or "") - kv_mgr = md.kv_cache_manager - index_head_dim = kv_mgr.index_head_dim - use_fp4 = getattr(kv_mgr, "use_fp4", False) - data_bytes_per_token = index_head_dim // 2 if use_fp4 else index_head_dim - scale_size = index_head_dim // kv_mgr.quant_block_size * 4 - tokens_per_block = md._tokens_per_block - block_stride = tokens_per_block * (data_bytes_per_token + scale_size) - width = min(md.gen_token_stride, md.kv_lens_cuda_2d.shape[1]) - expand = md.dsl_expand_factor if md.expand_for_dsl else 1 - num_sms = md.num_sms - block_offsets = md.indexer_k_cache_block_offsets - dense = bool(md.skip_indexer_for_gen_reqs) - num_tokens = md.num_tokens - dummy = md.kv_lens_cuda + if not correct and embed is None: + # Nothing of the native prologue to replace on this step. + return False + dummy = md.seq_lens_cuda if correct: pos = inputs["position_ids"] num_pos = inputs["input_ids"].shape[0] - md.num_ctx_tokens pos_off = preparer._buffers.previous_pos_id_offsets_cuda - kv_off = preparer._buffers.previous_kv_lens_offsets_cuda else: - pos = pos_off = kv_off = dummy + pos = pos_off = dummy num_pos = 1 if embed is not None: @@ -476,98 +231,62 @@ def target_prologue(preparer, inputs, *, enable_spec_decode: bool, runtime_draft ncb = 1 grid = (1,) - topk = md.topk_indices_buffer if dense else dummy - block_n = max(2, triton.next_power_of_2(num_tokens)) _target_prologue_kernel[grid]( - md.kv_lens_cuda, - kv_off, - md.seq_lens_cuda, pos, pos_off, - md.kv_lens_cuda_2d, - md.kv_lens_cuda_2d.stride(0), - md.scheduler_metadata_buffer, - md.scheduler_metadata_buffer_full_next_n, - md.kv_lens_expanded_cuda if expand > 1 else dummy, - md.scheduler_metadata_buffer_expanded if expand > 1 else dummy, - block_offsets, - block_offsets.stride(0), - md.req_idx_per_token, - md.slot_mapping_fp8, - md.slot_mapping_scale, - md.gen_kv_indptr, - md.gen_cached_token_indptr, - topk, - topk.stride(0) if dense else 0, ids, weight, emb_out, weight.shape[0] if embed is not None else 0, weight.stride(0) if embed is not None else 0, emb_out.stride(0) if embed is not None else 0, - num_tokens, N_POS=num_pos, - B=B, CORRECT=correct, - WIDTH=width, - EXPAND=expand, - NUM_SMS=num_sms, - SPLIT_KV=_draft_step._SPLIT_KV_BY_SMS[num_sms], - max_blocks=block_offsets.shape[1], - block_offsets_stride_1=block_offsets.stride(1), - tokens_per_block=tokens_per_block, - data_bytes_per_token=data_bytes_per_token, - scale_size=scale_size, - block_stride=block_stride, - scale_base_offset=tokens_per_block * data_bytes_per_token, - DENSE=dense, - TOPK=md.num_sparse_topk if dense else 1, EMBED=embed is not None, HIDDEN=hidden, NCB=ncb, - BLOCK_B=max(2, triton.next_power_of_2(B)), - BLOCK_W=max(2, triton.next_power_of_2(width)), - BLOCK_E=max(2, triton.next_power_of_2(B * expand)), - BLOCK_T=max(2, triton.next_power_of_2(1 + md.max_draft_tokens)), - BLOCK_SM=triton.next_power_of_2(num_sms + 1), BLOCK_P=max(2, triton.next_power_of_2(num_pos)), - BLOCK_N=block_n, - BLOCK_K=max(16, _DENSE_TILE // block_n), BLOCK_H=_BLOCK_H, num_warps=_NUM_WARPS, ) - # Host statements of the native sequence: on_update_kv_lens() (call A), - # the correction (device only), on_update_kv_lens() (call B). - _replay_host_on_update_kv_lens(md) - if correct: - _replay_host_on_update_kv_lens(md) - - if embed is not None: - setattr(md, _PACKET_ATTR, (inputs["input_ids"], weight, emb_out)) + if embed is not None and hook is not None: + hook.embedding_packet = (md, inputs["input_ids"], weight, emb_out) return True -def take_embedding(attn_metadata, input_ids: torch.Tensor, embed_tokens) -> Optional[torch.Tensor]: +def take_embedding( + attn_metadata, + input_ids: torch.Tensor, + embed_tokens, + hook: Optional["DecodePrologueHook"] = None, +) -> Optional[torch.Tensor]: """Return the target embedding rows the prologue launch produced, or None. - Consumes the step's packet. The rows are returned only for exactly the - ``input_ids`` tensor and the embedding weight the launch used; otherwise - the caller embeds natively (the launch's rows are then unused). + Consumes the packet ``hook`` holds. The rows are returned only for exactly + the attention metadata, ``input_ids`` tensor and embedding weight the + launch used; otherwise the caller embeds natively (the launch's rows are + then unused). Args: attn_metadata: the step's attention metadata. input_ids: ``[N]`` input ids the model forward embeds. embed_tokens: the model's ``Embedding`` module. + hook: the model's ``DecodePrologueHook`` the launch recorded on. Returns: ``[N, hidden]`` embedding rows, or None. """ - packet = vars(attn_metadata).pop(_PACKET_ATTR, None) + packet = None if hook is None else hook.embedding_packet if packet is None: return None - ids, weight, rows = packet - if ids is not input_ids or getattr(embed_tokens, "weight", None) is not weight: + hook.embedding_packet = None + md, ids, weight, rows = packet + if ( + md is not attn_metadata + or ids is not input_ids + or getattr(embed_tokens, "weight", None) is not weight + ): from ... import dispatch dispatch.log_capability("decode_prologue", "take_embedding", False, "packet does not match") @@ -580,9 +299,14 @@ class DecodePrologueHook: The engine calls it first in the step's input preprocessing, inside CUDA graph capture as well (``target_prologue``); the decode target then takes the rows that launch - embedded (``take_embedding``). + embedded (``take_embedding``). The rows of the last launch wait here, with the + attention metadata, ids and weight they were embedded for, until the forward + takes them. """ + def __init__(self) -> None: + self.embedding_packet: Optional[tuple] = None + def __call__( self, preparer, inputs, *, enable_spec_decode: bool, runtime_draft_len: int ) -> bool: @@ -591,9 +315,10 @@ def __call__( inputs, enable_spec_decode=enable_spec_decode, runtime_draft_len=runtime_draft_len, + hook=self, ) def take_embedding( self, attn_metadata, input_ids: torch.Tensor, embed_tokens ) -> Optional[torch.Tensor]: - return take_embedding(attn_metadata, input_ids, embed_tokens) + return take_embedding(attn_metadata, input_ids, embed_tokens, hook=self) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py index 4bb76882082b..be9165e80160 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py @@ -337,9 +337,13 @@ def step0_rows(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.T return self.gather_ids, rows def step_update( - self, attn_metadata, step: int, num_accepted_tokens, position_ids, gather_ids + self, attn_metadata, step: int, num_accepted_tokens, position_ids, gather_ids, refresh ) -> Optional["_draft_step.DraftStepResult"]: - """Fused per-step metadata update, or None for the native update.""" + """Fused per-step metadata update, or None for the native update. + + ``refresh`` refreshes the draft after its lengths changed + (``md.refresh_draft()`` of the running draft loop). + """ if not self.step_engaged: return None @@ -353,6 +357,7 @@ def run_draft_loop(): num_accepted_tokens, position_ids, gather_ids, + refresh, ) from ... import dispatch @@ -364,6 +369,7 @@ def run_draft_loop(): if _verify.verify_enabled() and reason is None: result = _verify.verified_step_update( run_draft_loop, + refresh, attn_metadata, step, self.batch_size, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py index 3092fd822773..f30f47577887 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py @@ -16,17 +16,19 @@ After each draft step the native one-model loop (``Eagle3OneModelWorker. _forward_linear_draft_loop``) advances the next step's position ids and the -KV lengths, then calls ``DSAtrtllmAttentionMetadata.update_for_spec_dec``, -which rebuilds the DSA decode metadata: the 2-D KV-length view, three DeepGEMM +KV lengths, then refreshes the draft (``md.refresh_draft()``), whose DSA +extension rebuilds the DSA decode metadata: the 2-D KV-length view, three DeepGEMM paged-MQA-logits schedules (each computed into a fresh tensor and copied into its persistent buffer), the expanded KV lengths for the CuTe DSL logits path, and the fused decode metadata (request index, indexer K-cache slot mappings and generation indptrs). That is about twelve small launches per step. -``draft_step_update`` performs the same update with one Triton launch plus the -native host-side bookkeeping. Every device tensor it writes holds exactly the -value the native chain writes. The schedule is the DeepGEMM SM100 metadata -partition (``deep_gemm/scheduler/sm100_paged_mqa_logits.cuh``): request work +``draft_step_update`` updates the core lengths through the core primitives and +``GlmTp8DSAMetadata.refresh_from_draft_lengths`` performs the DSA decode-state +refresh (plus the optional position-id advance) with one Triton launch. Every +device tensor it writes holds exactly the value the native chain writes. The +schedule is the DeepGEMM SM100 metadata partition +(``deep_gemm/scheduler/sm100_paged_mqa_logits.cuh``): request work ``ceil(ctx / SPLIT_KV) * next_n``, SM ``i`` starts at work unit ``i * q + min(i, rem)`` and records ``(request_q_token_start, work_in_request // next_n)``, with a ``(num_q_tokens, 0)`` sentinel past the @@ -35,12 +37,14 @@ """ from dataclasses import dataclass -from typing import Optional +from typing import Callable, Optional import torch import triton import triton.language as tl +from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata + _SCHEDULE_BLOCK_KV = 64 _SPLIT_KV_CANDIDATES = (64, 128, 256, 512) _MAX_BATCH = 64 @@ -81,12 +85,11 @@ def _write_schedule( @triton.jit def _draft_step_kernel( - kv_lens_ptr, # int32 [>= B] (in/out) - seq_lens_ptr, # int32 [>= B] (out when STEP0, else in) - num_accepted_ptr, # int32 [B] (STEP0) + kv_lens_ptr, # int32 [>= B] (in) + seq_lens_ptr, # int32 [>= B] (in) pos_in_ptr, # int32/int64 position ids of this step (HAS_POS) pos_out_ptr, # same dtype [B] (out, HAS_POS) - gather_ptr, # int64 [B] rows of pos_in to advance (STEP0 and HAS_POS) + gather_ptr, # int64 [B] rows of pos_in to advance (HAS_GATHER and HAS_POS) kv2d_ptr, # int32 [max_seqs, cap] (out) kv2d_stride, sched_ptr, # int32 [(NUM_SMS + 1) * 2] (out) @@ -102,9 +105,9 @@ def _draft_step_kernel( gen_cached_indptr_ptr, # int64 [>= B + 1] (out) num_tokens, B: tl.constexpr, - DRAFT_LEN: tl.constexpr, - STEP0: tl.constexpr, HAS_POS: tl.constexpr, + HAS_GATHER: tl.constexpr, + SKIP_SCHED: tl.constexpr, WIDTH: tl.constexpr, HAS_FULL: tl.constexpr, EXPAND: tl.constexpr, @@ -126,23 +129,13 @@ def _draft_step_kernel( offs_b = tl.arange(0, BLOCK_B) mask_b = offs_b < B - # ----- KV lengths and query lengths (the loop's in-place updates) ------- - kv_old = tl.load(kv_lens_ptr + offs_b, mask=mask_b, other=0) - if STEP0: - accepted = tl.load(num_accepted_ptr + offs_b, mask=mask_b, other=0) - kv = kv_old - (DRAFT_LEN - accepted) - seq = tl.full((BLOCK_B,), 1, tl.int32) - tl.store(seq_lens_ptr + offs_b, seq, mask=mask_b) - else: - kv = kv_old + 1 - seq = tl.load(seq_lens_ptr + offs_b, mask=mask_b, other=0) - tl.store(kv_lens_ptr + offs_b, kv, mask=mask_b) - kv = tl.where(mask_b, kv, 0) - seq = tl.where(mask_b, seq, 0) + # ----- KV lengths and query lengths (already updated by core primitives) -- + kv = tl.load(kv_lens_ptr + offs_b, mask=mask_b, other=0) + seq = tl.load(seq_lens_ptr + offs_b, mask=mask_b, other=0) # ----- next step's position ids ------------------------------------------ if HAS_POS: - if STEP0: + if HAS_GATHER: rows = tl.load(gather_ptr + offs_b, mask=mask_b, other=0) else: rows = offs_b.to(tl.int64) @@ -150,25 +143,24 @@ def _draft_step_kernel( tl.store(pos_out_ptr + offs_b, pos + 1, mask=mask_b) # ----- 2-D KV lengths and the paged-MQA-logits schedules ----------------- - offs_w = tl.arange(0, BLOCK_W) - kv2d_mask = mask_b[:, None] & (offs_w[None, :] < WIDTH) - tl.store( - kv2d_ptr + offs_b[:, None] * kv2d_stride + offs_w[None, :], - kv[:, None] + tl.zeros((BLOCK_B, BLOCK_W), tl.int32), - mask=kv2d_mask, - ) - _write_schedule(kv, mask_b, B, sched_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) - if HAS_FULL: - _write_schedule(kv, mask_b, B, sched_full_ptr, WIDTH, NUM_SMS, SPLIT_KV, BLOCK_SM) - if EXPAND > 1: - offs_e = tl.arange(0, BLOCK_E) - mask_e = offs_e < B * EXPAND - # repeat_interleave from registers (the updated lengths are not - # re-read from memory within this launch). - owner = (offs_e[:, None] // EXPAND) == offs_b[None, :] - kv_e = tl.sum(tl.where(owner, kv[None, :], 0), 1) - tl.store(kv_exp_ptr + offs_e, kv_e, mask=mask_e) - _write_schedule(kv_e, mask_e, B * EXPAND, sched_exp_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) + if not SKIP_SCHED: + offs_w = tl.arange(0, BLOCK_W) + kv2d_mask = mask_b[:, None] & (offs_w[None, :] < WIDTH) + tl.store( + kv2d_ptr + offs_b[:, None] * kv2d_stride + offs_w[None, :], + kv[:, None] + tl.zeros((BLOCK_B, BLOCK_W), tl.int32), + mask=kv2d_mask, + ) + _write_schedule(kv, mask_b, B, sched_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) + if HAS_FULL: + _write_schedule(kv, mask_b, B, sched_full_ptr, WIDTH, NUM_SMS, SPLIT_KV, BLOCK_SM) + if EXPAND > 1: + offs_e = tl.arange(0, BLOCK_E) + mask_e = offs_e < B * EXPAND + owner = (offs_e[:, None] // EXPAND) == offs_b[None, :] + kv_e = tl.sum(tl.where(owner, kv[None, :], 0), 1) + tl.store(kv_exp_ptr + offs_e, kv_e, mask=mask_e) + _write_schedule(kv_e, mask_e, B * EXPAND, sched_exp_ptr, 1, NUM_SMS, SPLIT_KV, BLOCK_SM) # ----- fused DSA decode metadata (same arithmetic as the native kernel) -- kv64 = kv.to(tl.int64) @@ -303,6 +295,145 @@ class DraftStepResult: position_ids: Optional[torch.Tensor] +class GlmTp8DSAMetadata(DSAMetadata): + """DSA metadata extension for ``glm_5_2_nvfp4__sm_103__tp8`` draft-loop refresh. + + Replaces the multi-launch DSA draft-length refresh with one Triton launch + when the step falls inside the fused pure-decode domain, and delegates to + ``DSAMetadata.refresh_from_draft_lengths`` otherwise. + """ + + fields = frozenset( + { + "_force_native_refresh", + "_pending_gather_ids", + "_pending_pos_in", + "_pending_pos_out", + } + ) + + adopts_target_state = True + draft_private = frozenset( + { + "_pending_pos_in", + "_pending_gather_ids", + "_pending_pos_out", + "_force_native_refresh", + } + ) + + _pending_pos_in: Optional[torch.Tensor] = None + _pending_gather_ids: Optional[torch.Tensor] = None + _pending_pos_out: Optional[torch.Tensor] = None + _force_native_refresh: bool = False + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self._pending_pos_in = None + self._pending_gather_ids = None + self._pending_pos_out = None + self._force_native_refresh = False + + def refresh_from_draft_lengths(self) -> None: + pos_in = self._pending_pos_in + gather_ids = self._pending_gather_ids + self._pending_pos_in = None + self._pending_gather_ids = None + self._pending_pos_out = None + md = self.metadata + if not self._force_native_refresh and ( + step_ineligibility(md, md.num_seqs, md.num_contexts) is None + ): + self._run_fused_refresh(pos_in, gather_ids) + return + if pos_in is not None: + from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids + + rows = gather_ids if gather_ids is not None else slice(0, md.num_seqs) + self._pending_pos_out = _select_mtp_position_ids(pos_in, rows) + 1 + super().refresh_from_draft_lengths() + + def _run_fused_refresh( + self, pos_in: Optional[torch.Tensor], gather_ids: Optional[torch.Tensor] + ) -> None: + md = self.metadata + batch_size = md.num_seqs + self._invalidate_length_prefixes() + if md.num_generations > 0: + self.max_gen_seq_len = md.seq_lens[md.num_contexts : md.num_seqs].max().item() + else: + self.max_gen_seq_len = 0 + skip_indexer_schedule = bool( + self.index_share.in_mtp_draft_loop and self.sparse_metadata_params.mtp_index_share + ) + self.indexer_schedule_stale = skip_indexer_schedule + self._invalidate_pool_view_cache() + if not skip_indexer_schedule: + self.gen_indexer_kv_lens_cuda_runtime = md.kv_lens_cuda[:batch_size] + self.kv_lens_row_reorder = None + + kv_mgr = md.kv_cache_manager + index_head_dim = kv_mgr.index_head_dim + use_fp4 = getattr(kv_mgr, "use_fp4", False) + data_bytes_per_token = index_head_dim // 2 if use_fp4 else index_head_dim + scale_size = index_head_dim // kv_mgr.quant_block_size * 4 + tokens_per_block = self._tokens_per_block + block_stride = tokens_per_block * (data_bytes_per_token + scale_size) + width = min(self.gen_token_stride, self.kv_lens_cuda_2d.shape[1]) + expand = self.dsl_expand_factor if self.expand_for_dsl else 1 + num_sms = self.num_sms + block_offsets = self.indexer_k_cache_block_offsets + has_pos = pos_in is not None + has_gather = has_pos and gather_ids is not None + pos_out = torch.empty_like(pos_in[:batch_size]) if has_pos else None + self._pending_pos_out = pos_out + dummy = md.kv_lens_cuda + + _draft_step_kernel[(1,)]( + md.kv_lens_cuda, + md.seq_lens_cuda, + pos_in if has_pos else dummy, + pos_out if has_pos else dummy, + gather_ids if has_gather else dummy, + self.kv_lens_cuda_2d, + self.kv_lens_cuda_2d.stride(0), + self.scheduler_metadata_buffer, + self.scheduler_metadata_buffer_full_next_n, + self.kv_lens_expanded_cuda if expand > 1 else dummy, + self.scheduler_metadata_buffer_expanded if expand > 1 else dummy, + block_offsets, + block_offsets.stride(0), + self.index_share.req_idx_per_token, + self.slot_mapping_fp8, + self.slot_mapping_scale, + self.gen_kv_indptr, + self.gen_cached_token_indptr, + md.num_tokens, + B=batch_size, + HAS_POS=has_pos, + HAS_GATHER=has_gather, + SKIP_SCHED=skip_indexer_schedule, + WIDTH=width, + HAS_FULL=True, + EXPAND=expand, + NUM_SMS=num_sms, + SPLIT_KV=_SPLIT_KV_BY_SMS[num_sms], + max_blocks=block_offsets.shape[1], + block_offsets_stride_1=block_offsets.stride(1), + tokens_per_block=tokens_per_block, + data_bytes_per_token=data_bytes_per_token, + scale_size=scale_size, + block_stride=block_stride, + scale_base_offset=tokens_per_block * data_bytes_per_token, + BLOCK_B=max(2, triton.next_power_of_2(batch_size)), + BLOCK_W=max(2, triton.next_power_of_2(width)), + BLOCK_E=max(2, triton.next_power_of_2(batch_size * expand)), + BLOCK_T=max(2, triton.next_power_of_2(1 + self.max_draft_tokens)), + BLOCK_SM=triton.next_power_of_2(num_sms + 1), + num_warps=_NUM_WARPS, + ) + + def step_ineligibility(md, batch_size: int, num_contexts: int) -> Optional[str]: """Return why the fused step update cannot replace the native chain, or None. @@ -312,21 +443,18 @@ def step_ineligibility(md, batch_size: int, num_contexts: int) -> Optional[str]: lengths, no expanded MTP buffers, no ragged verification, no FP4/FP8-DS MLA state and no dense top-k fallback. """ - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, - _fused_dsa_meta_enabled, - ) - - if not isinstance(md, DSAtrtllmAttentionMetadata): - return "metadata is not DSAtrtllmAttentionMetadata" - cls = type(md) - for name in ("update_for_spec_dec", "on_update_kv_lens", "on_update"): - if getattr(cls, name) is not getattr(DSAtrtllmAttentionMetadata, name): - return f"metadata overrides {name}" + from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata + from tensorrt_llm._torch.attention.backends.fp4_mla.state import Fp4MlaState from tensorrt_llm._torch.attention.backends.sparse.dsa import is_dsa_cache_manager - - # With a separate draft KV cache the loop has rebound the manager and the - # indexer buffers to the draft ones; the update reads them at call time. + from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import _fused_dsa_meta_enabled + + dsa = md.find_ext(GlmTp8DSAMetadata) + if type(dsa) is not GlmTp8DSAMetadata: + return "metadata does not carry GlmTp8DSAMetadata" + if md.find_ext(FlashMlaMetadata) is not None: + return "FlashMLA metadata state" + if md.find_ext(Fp4MlaState) is not None: + return "FP8-DS/FP4 MLA metadata state" if md.kv_cache_manager is None or not is_dsa_cache_manager(md.kv_cache_manager): return "KV cache manager is not a DSA cache manager" if num_contexts != 0 or md.num_contexts != 0: @@ -335,7 +463,7 @@ def step_ineligibility(md, batch_size: int, num_contexts: int) -> Optional[str]: return f"batch size {batch_size} outside the fused domain" if not _fused_dsa_meta_enabled(): return "native fused DSA metadata disabled" - if md.use_fp8_ds_mla or getattr(md, "fp4_mla_state", None) is not None: + if md.use_fp8_ds_mla: return "FP8-DS/FP4 MLA metadata state" if md.enable_ragged_verification or md.is_ragged_verify: return "ragged verification" @@ -369,6 +497,7 @@ def draft_step_update( num_accepted_tokens: Optional[torch.Tensor], position_ids: Optional[torch.Tensor], gather_ids: Optional[torch.Tensor], + refresh: Callable[[], None], ) -> Optional[DraftStepResult]: """Fused replacement of one draft step's metadata update, or None (native). @@ -376,7 +505,7 @@ def draft_step_update( position ids (``position_ids[gather_ids] + 1`` at step 0, ``position_ids[:batch_size] + 1`` later; skipped when ``position_ids`` is None), the step-0 switch to one query token per request, the KV-length - update and ``update_for_spec_dec``. + update and the draft refresh. Args: md: the draft loop's attention metadata. @@ -408,90 +537,24 @@ def draft_step_update( if position_ids is not None and (position_ids.dim() != 1 or not position_ids.is_contiguous()): return None - from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata - if step == 0: - # Host side of the native step-0 switch to one query token per request. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - if hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - md.num_contexts = 0 - - kv_mgr = md.kv_cache_manager - index_head_dim = kv_mgr.index_head_dim - use_fp4 = getattr(kv_mgr, "use_fp4", False) - data_bytes_per_token = index_head_dim // 2 if use_fp4 else index_head_dim - scale_size = index_head_dim // kv_mgr.quant_block_size * 4 - tokens_per_block = md._tokens_per_block - block_stride = tokens_per_block * (data_bytes_per_token + scale_size) - width = min(md.gen_token_stride, md.kv_lens_cuda_2d.shape[1]) - expand = md.dsl_expand_factor if md.expand_for_dsl else 1 - num_sms = md.num_sms - block_offsets = md.indexer_k_cache_block_offsets - has_pos = position_ids is not None - pos_out = torch.empty_like(position_ids[:batch_size]) if has_pos else None - dummy = md.kv_lens_cuda - - _draft_step_kernel[(1,)]( - md.kv_lens_cuda, - md.seq_lens_cuda, - num_accepted_tokens if step == 0 else dummy, - position_ids if has_pos else dummy, - pos_out if has_pos else dummy, - gather_ids if (step == 0 and has_pos) else dummy, - md.kv_lens_cuda_2d, - md.kv_lens_cuda_2d.stride(0), - md.scheduler_metadata_buffer, - md.scheduler_metadata_buffer_full_next_n, - md.kv_lens_expanded_cuda if expand > 1 else dummy, - md.scheduler_metadata_buffer_expanded if expand > 1 else dummy, - block_offsets, - block_offsets.stride(0), - md.req_idx_per_token, - md.slot_mapping_fp8, - md.slot_mapping_scale, - md.gen_kv_indptr, - md.gen_cached_token_indptr, - md.num_tokens, - B=batch_size, - DRAFT_LEN=runtime_draft_len, - STEP0=step == 0, - HAS_POS=has_pos, - WIDTH=width, - HAS_FULL=True, - EXPAND=expand, - NUM_SMS=num_sms, - SPLIT_KV=_SPLIT_KV_BY_SMS[num_sms], - max_blocks=block_offsets.shape[1], - block_offsets_stride_1=block_offsets.stride(1), - tokens_per_block=tokens_per_block, - data_bytes_per_token=data_bytes_per_token, - scale_size=scale_size, - block_stride=block_stride, - scale_base_offset=tokens_per_block * data_bytes_per_token, - BLOCK_B=max(2, triton.next_power_of_2(batch_size)), - BLOCK_W=max(2, triton.next_power_of_2(width)), - BLOCK_E=max(2, triton.next_power_of_2(batch_size * expand)), - BLOCK_T=max(2, triton.next_power_of_2(1 + md.max_draft_tokens)), - BLOCK_SM=triton.next_power_of_2(num_sms + 1), - num_warps=_NUM_WARPS, - ) - - # Host side of update_for_spec_dec -> on_update_kv_lens (no device work in - # the eligible domain, checked above). - TrtllmAttentionMetadata.update_for_spec_dec(md) - md.max_ctx_kv_len = 0 - md.num_ctx_cached_tokens = 0 - md.max_gen_seq_len = 1 - TrtllmAttentionMetadata.on_update_kv_lens(md) - md._invalidate_pool_view_cache() - md._fused_dsa_meta_armed = True - md.gen_indexer_kv_lens_cuda_runtime = md.get_indexer_kv_lens( - md.kv_lens_cuda[md.num_contexts : md.num_seqs] - ) - md._compute_kv_lens_row_reorder() - md.prepare_dense_topk_indices(md.kv_lens_cuda, device=True) + md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) + md.as_generation_rows() + md.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, + ) + if num_contexts > 0: + md.adjust_kv_lens(slice(0, num_contexts), 1) + else: + md.adjust_kv_lens(slice(0, batch_size), 1) + + md._pending_pos_in = position_ids + md._pending_gather_ids = gather_ids if step == 0 else None + md._pending_pos_out = None + refresh() + pos_out = md._pending_pos_out + md._pending_pos_out = None if step == 0: md.use_spec_decoding = False return DraftStepResult(position_ids=pos_out) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py index 3d7b0731b6b5..eec6373614a7 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py @@ -120,18 +120,26 @@ def _report_at_exit() -> None: def _device_fields(md) -> List[str]: - names = [] - for name, value in vars(md).items(): - if ( - isinstance(value, torch.Tensor) - and value.is_cuda - and 0 < value.numel() <= _MAX_SNAPSHOT_NUMEL - ): - names.append(name) + names = set() + owners = [vars(md)] + for ext in getattr(md, "_extensions", ()): + owners.append(vars(ext)) + for mapping in owners: + for name, value in mapping.items(): + if ( + not name.startswith("_") + and isinstance(value, torch.Tensor) + and value.is_cuda + and 0 < value.numel() <= _MAX_SNAPSHOT_NUMEL + and hasattr(md, name) + and getattr(md, name) is value + ): + names.add(name) return sorted(names) def _native_step_tail( + refresh, md, i, batch_size, @@ -144,37 +152,51 @@ def _native_step_tail( """The native per-step tail of ``Eagle3OneModelWorker._forward_linear_draft_loop``.""" from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids + from .draft_step import GlmTp8DSAMetadata + new_position_ids = None if position_ids is not None: rows = gather_ids if i == 0 else slice(0, batch_size) new_position_ids = _select_mtp_position_ids(position_ids, rows) + 1 - if i == 0: - md._seq_lens[:batch_size].fill_(1) - md._seq_lens_cuda[:batch_size].fill_(1) - md.on_update() - if hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - md.num_contexts = 0 - md.kv_lens_cuda[num_contexts:batch_size] -= ( - runtime_draft_len - num_accepted_tokens[num_contexts:] - ) - md.kv_lens_cuda[:num_contexts] += 1 - md.update_for_spec_dec() - md.use_spec_decoding = False - else: - md.kv_lens_cuda[:batch_size] += 1 - md.update_for_spec_dec() + dsa = md.find_ext(GlmTp8DSAMetadata) + if dsa is not None: + md._force_native_refresh = True + try: + if i == 0: + md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) + md.as_generation_rows() + md.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, + ) + md.adjust_kv_lens(slice(0, num_contexts), 1) + refresh() + md.use_spec_decoding = False + else: + md.adjust_kv_lens(slice(0, batch_size), 1) + refresh() + finally: + if dsa is not None: + md._force_native_refresh = False return new_position_ids def verified_step_update( - run_draft_loop, md, step, batch_size, draft_len, num_accepted_tokens, position_ids, gather_ids + run_draft_loop, + refresh, + md, + step, + batch_size, + draft_len, + num_accepted_tokens, + position_ids, + gather_ids, ): """Run the native tail, restore, run ``run_draft_loop()`` and count differences.""" names = _device_fields(md) before = {name: getattr(md, name).clone() for name in names} native_pos = _native_step_tail( - md, step, batch_size, 0, draft_len, num_accepted_tokens, position_ids, gather_ids + refresh, md, step, batch_size, 0, draft_len, num_accepted_tokens, position_ids, gather_ids ) after_native = {name: getattr(md, name).clone() for name in names} for name in names: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py index e6c1800b0f16..ed0874744c8d 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py @@ -49,6 +49,16 @@ _ROPE_DIM = 64 +def _dsa_state(metadata): + """The DSA extension of ``metadata``, which holds the grouped-remap state. + + Imported on use, as the rest of this module's attention imports. + """ + from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata + + return metadata.ext(DSAMetadata) + + @dataclass class IndexerKScatter: """Deferred native indexer K-cache scatter of one layer and one forward. @@ -463,7 +473,7 @@ def remap_topk_to_global( Follows the native grouping rules: the leader of an active full+shared group computes the whole group in one launch and publishes it in - ``metadata._group_remap_batched``; followers slice it; everything else + the DSA metadata's ``_group_remap_batched``; followers slice it; everything else remaps per layer. """ from tensorrt_llm._torch.attention.backends.sparse.dsa import backend as dsa_backend @@ -473,7 +483,7 @@ def remap_topk_to_global( copy = packet.copy local_layer_idx = backend.get_local_layer_idx(metadata) if dsa_backend._GROUP_REMAP and not metadata.in_mtp_draft_loop: - struct = metadata._ensure_group_remap_struct() + struct = _dsa_state(metadata)._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is not None and local_layer_idx < len(leader_of): leader = leader_of[local_layer_idx] @@ -494,9 +504,9 @@ def remap_topk_to_global( page_index_scale=struct["group_scale"][leader], copy=copy, ) - metadata._group_remap_batched[leader] = batched + _dsa_state(metadata)._group_remap_batched[leader] = batched return batched[slot] - batched = metadata._group_remap_batched.get(leader, None) + batched = _dsa_state(metadata)._group_remap_batched.get(leader, None) if ( copy is None and batched is not None diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py index 5c74eff17215..1d907042f4c1 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py @@ -74,8 +74,14 @@ def make_runtime_hooks(model: "GlmMoeDsaForCausalLM") -> RuntimeHooks: # The decode target takes the embedding rows of the prologue's launch from the # same object (the core's ``decode_prologue``). decode_prologue = model.model.decode_prologue = DecodePrologueHook() + metadata_extensions = () + if plan.enabled("draft_loop"): + from .kernels.draft_loop.draft_step import GlmTp8DSAMetadata + + metadata_extensions = (GlmTp8DSAMetadata,) return RuntimeHooks( loop_on_execution_stream=plan.enabled("execution_stream_loop"), decode_prologue=decode_prologue, executor_events=GlmExecutorEvents(plan), + metadata_extensions=metadata_extensions, ) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/spec_worker.py index 3356edff361a..83ba7979a5a8 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/spec_worker.py @@ -23,9 +23,9 @@ import torch import torch.nn.functional as F -from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareMetadata +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManager -from tensorrt_llm._torch.speculative.eagle3 import MTPEagleWorker, _reset_mtp_index_share +from tensorrt_llm._torch.speculative.eagle3 import MTPEagleWorker from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids from . import dispatch @@ -157,54 +157,51 @@ def _forward_impl( batch_size=batch_size, ) else: - # The step-0 sequence lengths, before the draft loop rewrites them. + # The step-0 sequence lengths, before the draft loop changes them: the target's, + # which the draft loop never changes when it runs on the draft metadata. pre_draft_seq_lens = attn_metadata._seq_lens_cuda - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - if draft_loop_ops is not None: - inputs = draft_loop_ops.first_drafter_inputs(position_ids, hidden_states) - else: - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) - - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here - # so the post-loop restore (below) can put attn_metadata back into a - # state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # The draft forwards run on the draft metadata, which leaves the target's + # untouched; without one, the target's state is saved and restored around the + # draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + if draft_loop_ops is not None: + inputs = draft_loop_ops.first_drafter_inputs(position_ids, hidden_states) + else: + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - next_draft_tokens, loop_next_new_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - accepted_tokens, - pre_draft_seq_lens, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here + # so the post-loop restore (below) can put attn_metadata back into a + # state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + + next_draft_tokens, loop_next_new_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + accepted_tokens, + pre_draft_seq_lens, + ) # prepare next new tokens to support overlap scheduler if draft_loop_ops is not None and draft_loop_ops.next_new_tokens is not None: @@ -278,8 +275,6 @@ def _forward_linear_draft_loop( accepted_tokens=accepted_tokens, pre_draft_seq_lens=pre_draft_seq_lens, ) - from tensorrt_llm._torch.attention.backends.sparse.dsa import is_dsa_cache_manager - runtime_draft_len = spec_metadata.runtime_draft_len num_gens = batch_size - num_contexts next_draft_tokens = [] @@ -293,9 +288,7 @@ def _forward_linear_draft_loop( # Sparse backends that can reuse one indexer selection across the draft # loop expose this setter; each indexer still decides whether its own # config turns the reuse on. - uses_mtp_index_share = self.is_mtp_eagle and isinstance( - attn_metadata, MTPIndexShareMetadata - ) + index_share = attn_metadata.find_ext(MTPIndexShareExtension) if self.is_mtp_eagle else None # The draft tail (draft_tail): one launch for the draft LM head, the vocab-shard max # and the cross-rank argmax. Decided once per forward, before any device work, # identically on every TP rank. @@ -318,32 +311,25 @@ def _forward_linear_draft_loop( self, draft_model, spec_metadata, draft_loop_ops ) with contextlib.ExitStack() as draft_scope: - if uses_mtp_index_share: - attn_metadata.set_in_mtp_draft_loop(True) + if index_share is not None: + index_share.set_in_mtp_draft_loop(True) # Accepted counts let the indexer stash each gen's last-accepted row. - attn_metadata.set_mtp_num_accepted(num_accepted_tokens) + index_share.set_mtp_num_accepted(num_accepted_tokens) # The metadata outlives this call, so clear the state even when a # draft step raises: a stale draft-loop flag would make the next # target forward reuse these selections. The loop writes - # set_skip_topk on the metadata yielded below, which is this same - # object: draft_kv_cache_context swaps buffers in place for every - # TrtllmAttentionMetadata, and sparse metadata all derive from it. - draft_scope.callback(_reset_mtp_index_share, attn_metadata) + # set_skip_topk on the extension of the metadata yielded below, + # which is this same object: draft_kv_cache_context yields a + # draft metadata as it is. + draft_scope.callback(index_share.reset_mtp_index_share) draft_attn_metadata = draft_scope.enter_context( self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager) ) attn_metadata = draft_attn_metadata inputs["attn_metadata"] = draft_attn_metadata - if self.is_mtp_eagle and is_dsa_cache_manager(draft_kv_cache_manager): - # Overlap scheduling corrects kv_lens_cuda from the runtime - # accepted-token counts inside the captured graph. The target - # forward refreshes target slot mappings during that correction; - # refresh once more after rebinding so the first draft forward - # writes the separate DSA indexer cache at the same positions. - attn_metadata.on_update_kv_lens() for i in range(runtime_draft_len): - if uses_mtp_index_share: - attn_metadata.set_skip_topk(i > 0) + if index_share is not None: + index_share.set_skip_topk(i > 0) # Run draft model (mode-specific via helper). The helper # passes ``all_rank_num_tokens`` as a kwarg so the draft model # handles save/restore internally (Eagle3DraftModel.forward @@ -545,6 +531,7 @@ def _forward_linear_draft_loop( num_accepted_tokens, inputs["position_ids"] if i + 1 < runtime_draft_len else None, gather_ids if i == 0 else None, + self.refresh_draft, ) if i + 1 < runtime_draft_len: if self.is_mtp_eagle: @@ -569,26 +556,23 @@ def _forward_linear_draft_loop( if draft_step is not None: pass # The draft-loop step update updated the metadata above. elif i == 0: - attn_metadata._seq_lens[:batch_size].fill_(1) - attn_metadata._seq_lens_cuda[:batch_size].fill_(1) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(0, batch_size), host=1, device=1, update_counts=True + ) has_kv_cache = inputs["attn_metadata"].kv_cache_manager is not None - if has_kv_cache and hasattr(attn_metadata, "host_request_types"): - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) - if has_kv_cache: - attn_metadata.num_contexts = 0 - if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - runtime_draft_len - num_accepted_tokens[num_contexts:] - ) - attn_metadata.kv_lens_cuda[:num_contexts] += 1 - if has_kv_cache: + # The context requests continue as one-token generation rows. + attn_metadata.as_generation_rows() self._prepare_flash_mla_generation_layout( attn_metadata, num_contexts, batch_size ) if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.update_for_spec_dec() + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, + ) + attn_metadata.adjust_kv_lens(slice(0, num_contexts), 1) + self.refresh_draft() # Both Eagle3 and MTP Eagle drafters take ``draft_len + 1`` # tokens in the first draft step (attention runs in spec-dec @@ -598,8 +582,8 @@ def _forward_linear_draft_loop( attn_metadata.use_spec_decoding = False else: if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[:batch_size] += 1 - attn_metadata.update_for_spec_dec() + attn_metadata.adjust_kv_lens(slice(0, batch_size), 1) + self.refresh_draft() stacked = draft_loop_ops.finalize(next_draft_tokens) if draft_loop_ops is not None else None next_draft_tokens = ( stacked if stacked is not None else torch.stack(next_draft_tokens, dim=1) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py index 61f1bc9126f6..2d61536381c8 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py @@ -15,7 +15,10 @@ MLAParams, PositionalEmbeddingParams, ) -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( + DSAMetadata, + DSAtrtllmAttentionMetadata, +) from tensorrt_llm._torch.attention.backends.sparse.dsa.params import DSAParams from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm.models.modeling_utils import QuantConfig @@ -26,6 +29,12 @@ transform_local_topk_and_prepare_pool_view_grouped, ) + +def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: + """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" + return metadata.ext(DSAMetadata) + + _GROUP_REMAP = os.environ.get("TRTLLM_DISABLE_DSA_GROUP_REMAP", "0") != "1" @@ -182,7 +191,8 @@ def _grouped_remap_topk_to_global( that equivalence is covered by unit tests (see ``tests/unittest/_torch/attention/sparse/dsa/test_cpp_custom_ops.py``). """ - struct = metadata._ensure_group_remap_struct() + dsa_state = _dsa_state(metadata) + struct = dsa_state._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is None or local_layer_idx >= len(leader_of): return None @@ -200,7 +210,7 @@ def _grouped_remap_topk_to_global( struct["group_scale"][leader], is_generation, ) - metadata._group_remap_batched[leader] = batched + dsa_state._group_remap_batched[leader] = batched else: # Shared layer: read the leader's precomputed batch. The batch is # cleared at every step boundary, so a present entry is from this @@ -208,7 +218,7 @@ def _grouped_remap_topk_to_global( # top-k shape, and otherwise fall back to the per-layer path. This # makes any leader/follower ordering or shape mismatch a safe # per-layer fallback rather than a stale/out-of-bounds read. - batched = metadata._group_remap_batched.get(leader, None) + batched = dsa_state._group_remap_batched.get(leader, None) if ( batched is None or slot >= batched.shape[0] diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py index 7b2d5e954204..2a28a41d94b2 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py @@ -13,7 +13,10 @@ import torch.nn.functional as F from tensorrt_llm._torch.attention.backends.interface import MLAParams, PositionalEmbeddingParams -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( + DSAMetadata, + DSAtrtllmAttentionMetadata, +) from tensorrt_llm._torch.attention.backends.sparse.dsa.params import DSAParams from tensorrt_llm._torch.attention.backends.sparse.params import ( get_indexer_mqa_logits_elem_budget, @@ -36,6 +39,12 @@ from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig + +def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: + """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" + return metadata.ext(DSAMetadata) + + _INDEXER_MQA_LOGITS_ELEM_BUDGET = get_indexer_mqa_logits_elem_budget() @@ -58,34 +67,35 @@ def transform_local_topk_and_prepare_pool_view( ) -> Tuple[torch.Tensor, torch.Tensor]: """Convert local topk indices to global pool indices and prepare KV pool. - Uses cached values from attn_metadata._ensure_pool_view_cached() + Uses the pool-view values the DSA metadata caches (DSAMetadata._ensure_pool_view_cached) to avoid redundant Python/CUDA overhead across layers. """ + dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - attn_metadata._ensure_pool_view_cached() + dsa._ensure_pool_view_cached() page_index_scale, layer_offset = ( attn_metadata.kv_cache_manager.get_primary_pool_page_index_params(layer_idx) ) if is_generation: - block_table = attn_metadata._cached_block_table_gen - req_idx = attn_metadata._cached_req_idx_gen + block_table = dsa._cached_block_table_gen + req_idx = dsa._cached_req_idx_gen else: - block_table = attn_metadata._cached_block_table_ctx - req_idx = attn_metadata._cached_req_idx_ctx + block_table = dsa._cached_block_table_ctx + req_idx = dsa._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global( req_idx, block_table, topk_indices, - attn_metadata._cached_tokens_per_block, + dsa._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * attn_metadata._cached_tokens_per_block, + page_index_scale * dsa._cached_tokens_per_block, layer_offset, ) - return global_indices, attn_metadata._cached_pool_view + return global_indices, dsa._cached_pool_view def transform_local_topk_and_prepare_pool_view_grouped( @@ -110,24 +120,25 @@ def transform_local_topk_and_prepare_pool_view_grouped( is bit-identical to ``transform_local_topk_and_prepare_pool_view`` for the member with offset ``layer_ids[z]``. MLA-only path. """ + dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - attn_metadata._ensure_pool_view_cached() + dsa._ensure_pool_view_cached() if is_generation: - block_table = attn_metadata._cached_block_table_gen - req_idx = attn_metadata._cached_req_idx_gen + block_table = dsa._cached_block_table_gen + req_idx = dsa._cached_req_idx_gen else: - block_table = attn_metadata._cached_block_table_ctx - req_idx = attn_metadata._cached_req_idx_ctx + block_table = dsa._cached_block_table_ctx + req_idx = dsa._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global_grouped( req_idx, block_table, topk_indices, - attn_metadata._cached_tokens_per_block, + dsa._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * attn_metadata._cached_tokens_per_block, + page_index_scale * dsa._cached_tokens_per_block, layer_ids, ) @@ -987,9 +998,10 @@ def _store_mtp_shared_rows( ctx_rows = topk_indices_buffer[ctx_last] rows = ctx_rows if rows is None else torch.cat([ctx_rows, rows]) if rows is not None: - shared_topk_indices = metadata.shared_topk_indices + index_share = _dsa_state(metadata).index_share + shared_topk_indices = index_share.shared_topk_indices if shared_topk_indices is None: - metadata.shared_topk_indices = rows.contiguous() + index_share.bind_buffers(shared_topk_indices=rows.contiguous()) else: row_start = num_contexts if is_generation else 0 shared_topk_indices[row_start : row_start + rows.shape[0], : rows.shape[1]].copy_( diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_mtp_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_mtp_worker.py index cac11254463e..5a72800bbf5c 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_mtp_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_mtp_worker.py @@ -9,10 +9,9 @@ import torch -from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareMetadata +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.speculative import MTPEagleWorker -from tensorrt_llm._torch.speculative.eagle3 import _reset_mtp_index_share from tensorrt_llm._torch.speculative.interface import ( _FORCE_ACCEPT_RNG_COUNTER_STRIDE, _FORCE_ACCEPT_RNG_POOL_SIZE, @@ -288,7 +287,7 @@ def _small_batch_target( and not spec_metadata.wants_advanced_draft_sampling and not spec_metadata.is_ragged_verify and not torch.compiler.is_compiling() - and isinstance(attn_metadata, MTPIndexShareMetadata) + and attn_metadata.find_ext(MTPIndexShareExtension) is not None and attn_metadata.num_tokens == rows and ids.dtype == torch.int32 and tuple(ids.shape) == (rows,) @@ -344,8 +343,6 @@ def _forward_small_batch_draft_loop( None means only the preflight declined: no draft forward has executed. Once the first layer starts, failures propagate without replaying it. """ - from tensorrt_llm._torch.attention.backends.sparse.dsa import is_dsa_cache_manager - draft_len = spec_metadata.runtime_draft_len first_rows = batch_size * (draft_len + 1) accepted_tokens = inputs["input_ids"].view(batch_size, draft_len + 1) @@ -354,10 +351,11 @@ def _forward_small_batch_draft_loop( selector = FirstDraftRowSelect(batch_indices, num_accepted, draft_len + 1) # Do not modify the caller's inputs if the rebound-manager preflight declines. inputs = dict(inputs) + index_share = attn_metadata.ext(MTPIndexShareExtension) with contextlib.ExitStack() as scope: - attn_metadata.set_in_mtp_draft_loop(True) - attn_metadata.set_mtp_num_accepted(num_accepted_tokens) - scope.callback(_reset_mtp_index_share, attn_metadata) + index_share.set_in_mtp_draft_loop(True) + index_share.set_mtp_num_accepted(num_accepted_tokens) + scope.callback(index_share.reset_mtp_index_share) attn_metadata = scope.enter_context( self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager) ) @@ -365,15 +363,13 @@ def _forward_small_batch_draft_loop( # separately configured draft cache. Validate the actual rebound views. if not helper.supports(attn_metadata, first_rows, batch_size): return None - if is_dsa_cache_manager(draft_kv_cache_manager): - attn_metadata.on_update_kv_lens() inputs["attn_metadata"] = attn_metadata inputs["owned_draft"] = helper inputs["row_select"] = selector draft_tokens = accepted_tokens.new_empty((batch_size, draft_len)) next_new_tokens = None for step in range(draft_len): - attn_metadata.set_skip_topk(step > 0) + index_share.set_skip_topk(step > 0) hidden, _ = self._run_draft_forward(draft_model, inputs, spec_metadata, step) # The layer already selected both hidden and residual BEFORE MoE. # A second gather here would select the wrong request (or go OOB). @@ -381,9 +377,10 @@ def _forward_small_batch_draft_loop( raise RuntimeError("small-batch MTP did not select its first-step rows") logits, hidden_half = helper.head(hidden, next_step=step + 1 < draft_len) if step == 0: - attn_metadata._seq_lens[:batch_size].fill_(1) - attn_metadata.on_update() - attn_metadata.num_contexts = 0 + # One row per request from here on, on the host; the step has no + # context requests to turn into generation ones. + attn_metadata.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + lengths = attn_metadata.lengths_for_kernel_update() next_projection, next_positions, next_new_tokens = helper.sample_step( logits, hidden, @@ -391,16 +388,16 @@ def _forward_small_batch_draft_loop( step, inputs["position_ids"], selector.gather_ids if step == 0 else None, - attn_metadata.seq_lens_cuda[:batch_size], - attn_metadata.kv_lens_cuda[:batch_size], + lengths["seq_lens_cuda"][:batch_size], + lengths["kv_lens_cuda"][:batch_size], num_accepted, accepted_tokens, batch_indices, hidden_half=hidden_half, ) - # The fused op just wrote seq/KV lengths. Main owns derived DSA - # slots, schedules and dense selection; update even the final step. - attn_metadata.update_for_spec_dec() + # The fused op just wrote seq/KV lengths. The metadata re-derives the DSA + # slots, schedules and dense selection from them, even after the final step. + self.refresh_draft() if step == 0: attn_metadata.use_spec_decoding = False if step + 1 < draft_len: @@ -483,7 +480,7 @@ def _draft_on_target( Each step's last kernel writes the next step's sequence and KV lengths in place. The steps after the first run no indexer and read nothing else of the DSA decode metadata, - so the built-in loop's per-step rebuild of it (``update_for_spec_dec``) is left out; + so the built-in loop's per-step rebuild of it (``md.refresh_draft()``) is left out; with a context short enough that the metadata skips the indexer, only the next step's dense selection is refreshed from the new lengths. The host side of draft step 0's switch to one row per request comes before that. @@ -496,22 +493,23 @@ def _draft_on_target( draft_tokens = accepted_tokens.new_empty((batch_size, draft_len)) positions = inputs["position_ids"] eh_proj_output = gather_ids = pool_rows = next_new_tokens = None - uses_mtp_index_share = isinstance(attn_metadata, MTPIndexShareMetadata) + index_share = attn_metadata.find_ext(MTPIndexShareExtension) with contextlib.ExitStack() as draft_scope: - if uses_mtp_index_share: - attn_metadata.set_in_mtp_draft_loop(True) - attn_metadata.set_mtp_num_accepted(num_accepted_tokens) - draft_scope.callback(_reset_mtp_index_share, attn_metadata) + if index_share is not None: + index_share.set_in_mtp_draft_loop(True) + index_share.set_mtp_num_accepted(num_accepted_tokens) + draft_scope.callback(index_share.reset_mtp_index_share) attn_metadata = draft_scope.enter_context( self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager) ) - # The draft loop's copy of the sequence lengths (prepare_for_spec_dec) and the KV - # lengths, which the draft-step kernels advance. - seq_lens = attn_metadata.seq_lens_cuda[:batch_size] - kv_lens = attn_metadata.kv_lens_cuda[:batch_size] + # The draft metadata's sequence and KV lengths, which the draft-step kernels advance. + lengths = attn_metadata.lengths_for_kernel_update() + seq_lens = lengths["seq_lens_cuda"][:batch_size] + kv_lens = lengths["kv_lens_cuda"][:batch_size] + attn_metadata.clear_lengths_changed() for i in range(draft_len): - if uses_mtp_index_share: - attn_metadata.set_skip_topk(i > 0) + if index_share is not None: + index_share.set_skip_topk(i > 0) if i == 0: hidden_states, gather_ids, pool_rows = decode.draft_first_step( attn_metadata, @@ -532,8 +530,7 @@ def _draft_on_target( if i == 0: # One row per request from here on, on the host (no context requests to # turn into generation ones); the kernel below fills seq_lens. - attn_metadata._seq_lens[:batch_size].fill_(1) - attn_metadata.on_update() + attn_metadata.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) if last: next_new_tokens = decode.draft_sample_glue( logits, @@ -562,6 +559,10 @@ def _draft_on_target( attn_metadata.prepare_dense_topk_indices( attn_metadata.kv_lens_cuda, device=True ) + # A deliberate exception to the full draft refresh: the later + # steps read no other length-derived state (see the docstring), + # so the dense selection above is the whole refresh. + attn_metadata.clear_lengths_changed() if i == 0: attn_metadata.use_spec_decoding = False self._drafted_rows = (draft_tokens, next_new_tokens) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py index 9fe9802ab2c3..1916a6fa9073 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py @@ -167,7 +167,8 @@ def _k3_accept_step_applies( if ( self._ctx_block_tables is None or self._ctx_pool_windowed - or getattr(attn_metadata, "draft_kv_cache_block_offsets", None) is None + or attn_metadata.find("draft_or_none", None) is None + or attn_metadata.draft.kv_cache_block_offsets is None or getattr(attn_metadata, "kv_lens_cuda", None) is None ): return False @@ -431,11 +432,16 @@ def _on_acceptance( ) -> None: """Called with this step's acceptance when ``k3_spec_accept`` produced it (a hook for drafter families).""" - def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=None): + def _k3_accept( + self, logits, attn_metadata, spec_metadata, draft_model, draft_kv_lens, shard=None + ): """``trtllm::k3_spec_accept`` for a step ``_k3_accept_applies`` accepted: the acceptance, the block table, - the KDA replay record, kv_lens + 1 and the drafter's inputs in one launch. ``shard``: the exchange - workspace and first column when ``logits`` are this rank's vocabulary shard (``_k3_logits_shard``). - Returns (accepted tokens, num accepted, {rewind, bonus, qpos, cpos, noise}).""" + the KDA replay record, the drafter's KV lengths + 1 and the drafter's inputs in one launch. + + ``draft_kv_lens`` are the draft metadata's device KV lengths, which hold the target's at the start of the + draft loop; the kernel advances them, never the target's. ``shard``: the exchange workspace and first + column when ``logits`` are this rank's vocabulary shard (``_k3_logits_shard``). Returns (accepted tokens, + num accepted, {rewind, bonus, qpos, cpos, noise}).""" from tensorrt_llm._torch._experimental.modeling_v2.models.kimi_k3_vl.kernels.k3_spec_accept import ( op as accept_op, ) @@ -459,10 +465,10 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No pool, counter = dummy_rng embed = draft_model.draft_model_full.model.embed_tokens assert self._batch_to_slot_buffer is not None - accepted, num_acc, rewind, bonus, qpos, cpos, noise = torch.ops.trtllm.k3_spec_accept( + accepted, num_acc, _rewind, bonus, qpos, cpos, noise = torch.ops.trtllm.k3_spec_accept( logits, spec_metadata.draft_tokens.reshape(num_gens, K), - attn_metadata.draft_kv_cache_block_offsets, + attn_metadata.draft.kv_cache_block_offsets, self._ctx_pool_idx, self._ctx_block_divisor, self._ctx_block_counts, @@ -470,7 +476,7 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No mgr._kda_replay.prev_num_accepted_tokens, attn_metadata.mamba_metadata.state_indices, mgr._dummy_request_mask, - attn_metadata.kv_lens_cuda, + draft_kv_lens, self._batch_to_slot_buffer.tensor, self._ctx_len, self._max_ctx, @@ -486,7 +492,7 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No return ( accepted, num_acc, - dict(rewind=rewind, bonus=bonus, qpos=qpos, cpos=cpos, noise=noise), + dict(bonus=bonus, qpos=qpos, cpos=cpos, noise=noise), ) @staticmethod @@ -586,208 +592,208 @@ def _forward_impl( self._execute_guided_decoder_if_present(logits) - prep = None - if fused_accept: - # Saved before the kernel updates kv_lens (warmup also snapshots kv_lens_cuda here). - self._prepare_attn_metadata_for_dflash(attn_metadata, spec_metadata) - accepted_tokens, num_accepted_tokens, prep = self._k3_accept( - logits, attn_metadata, spec_metadata, draft_model, shard - ) - else: - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata - ) + # The drafter's inputs set their lengths on the draft metadata, which leaves the target's untouched. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + prep = None + if fused_accept: + accepted_tokens, num_accepted_tokens, prep = self._k3_accept( + logits, + attn_metadata, + spec_metadata, + draft_model, + # The kernel advances the draft's KV lengths: the buffer comes from its core. + draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], + shard, + ) + else: + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + logits, attn_metadata, spec_metadata + ) - # Opt-in acceptance recording (env-gated; eager-mode measurement - # runs only). Skipped for CUDA-graph batches (capture/replay/warmup - # use synthetic requests and forbid the host sync). - if ( - self._accept_stats is not None - and num_gens > 0 - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - ): - self._accept_stats.on_accept( - spec_metadata.request_ids[num_contexts:batch_size], - num_accepted_tokens[num_contexts:batch_size].tolist(), - ) + # Opt-in acceptance recording (env-gated; eager-mode measurement + # runs only). Skipped for CUDA-graph batches (capture/replay/warmup + # use synthetic requests and forbid the host sync). + if ( + self._accept_stats is not None + and num_gens > 0 + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + ): + self._accept_stats.on_accept( + spec_metadata.request_ids[num_contexts:batch_size], + num_accepted_tokens[num_contexts:batch_size].tolist(), + ) - if fused_accept: - # The kernel recorded the KDA replay acceptance and advanced kv_lens_cuda. - self._kv_rewind_amount = prep["rewind"] - self._kv_rewind_nc = num_contexts - self._kv_rewind_bs = batch_size - self._kv_rewind_pending = True - attn_metadata.update_for_spec_dec() - else: - # Update GDN/Mamba recurrent states to the accepted token's state. - if num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): + if fused_accept: + # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft + # metadata re-derives its length-dependent state from them. + self.refresh_draft() + elif num_gens > 0 and isinstance( + attn_metadata.kv_cache_manager, MambaHybridCacheManager + ): + # Update GDN/Mamba recurrent states to the accepted token's state. attn_metadata.kv_cache_manager.update_mamba_states( attn_metadata=attn_metadata, num_accepted_tokens=num_accepted_tokens, state_indices=attn_metadata.mamba_metadata.state_indices, ) - self._prepare_attn_metadata_for_dflash(attn_metadata, spec_metadata) - self._prepare_kv_for_draft_forward( - attn_metadata, num_accepted_tokens, num_contexts, batch_size - ) - - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens + ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + prep=prep, ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, - prep=prep, - ) - if num_gens > 0: - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), - ) + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_hidden_states = self._draft_block_hidden_states( - draft_model, hidden_states_out, num_gens, block_size, K - ) - gen_logits = self._draft_block_logits( - draft_model, gen_hidden_states, attn_metadata, spec_metadata - ) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_hidden_states = self._draft_block_hidden_states( + draft_model, hidden_states_out, num_gens, block_size, K + ) + gen_logits = self._draft_block_logits( + draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata + ) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with one - # coherent path through the block (absent for plain DFlash). - if getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( - draft_model, + # DFlash 2: replace the independent per-position picks with one + # coherent path through the block (absent for plain DFlash). + if getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( + draft_model, + gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + spec_metadata, + ) + vocab_size = gen_logits.shape[-1] + + gen_draft_tokens = self.sample_draft_tokens( gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], spec_metadata, + batch_size, + num_contexts=num_contexts, ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None here -> no calibration rows collected; the - # DSpark confidence-scheduled verification MR supplies the - # real provider, see accept_stats.py). Guarded off for - # CUDA-graph batches and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], - draft_model, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - gen_draft_tokens, - ) + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None here -> no calibration rows collected; the + # DSpark confidence-scheduled verification MR supplies the + # real provider, see accept_stats.py). Guarded off for + # CUDA-graph batches and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], + draft_model, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs( + spec_metadata, num_contexts, num_gens, K, gen_vocab + ) - self._restore_attn_metadata_from_spec_dec(attn_metadata) - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._ctx_len.copy_(self._saved_ctx_len) - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False - - outputs = { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } - if shard is not None: - # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. - outputs["logits_vocab_shard"] = True - return outputs + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._ctx_len.copy_(self._saved_ctx_len) + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False + + outputs = { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } + if shard is not None: + # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. + outputs["logits_vocab_shard"] = True + return outputs def _draft_block_hidden_states( self, @@ -1045,10 +1051,12 @@ def prepare_1st_drafter_inputs( # Update seq_lens for gen requests to K+1 (the number of tokens # the target forward actually processed). - attn_metadata._seq_lens_cuda[num_contexts : num_contexts + num_gens] = ( - total_tokens_per_req + attn_metadata.set_seq_lens( + slice(num_contexts, num_contexts + num_gens), + host=total_tokens_per_req, + device=total_tokens_per_req, ) - attn_metadata._seq_lens[num_contexts : num_contexts + num_gens] = total_tokens_per_req + self.refresh_draft() else: noise_embedding = hidden_states.new_empty(0, 0, hidden_dim) query_positions = torch.empty(0, 0, dtype=torch.long, device="cuda") @@ -1221,10 +1229,9 @@ def sample_and_accept_draft_tokens(self, logits, attn_metadata, spec_metadata): def _on_acceptance( self, accepted_tokens, num_accepted_tokens, attn_metadata, spec_metadata ) -> None: - """Keeps this step's acceptance for ``k3_markov``'s next_new_tokens (and the metadata whose KV lengths it - rewinds).""" + """Keeps this step's acceptance for ``k3_markov``'s next_new_tokens.""" self._k3_acceptance = ( - accepted_tokens, num_accepted_tokens, attn_metadata.num_contexts, spec_metadata, attn_metadata + accepted_tokens, num_accepted_tokens, attn_metadata.num_contexts, spec_metadata ) # fmt: skip def _k3_markov_chain( @@ -1247,21 +1254,8 @@ def _k3_markov_chain( acceptance = getattr(self, "_k3_acceptance", None) if acceptance is None: raise RuntimeError("DSpark Markov head: k3_markov needs this step's acceptance first") - accepted_tokens, num_accepted_tokens, num_contexts, spec_metadata, attn_metadata = ( - acceptance - ) + accepted_tokens, num_accepted_tokens, num_contexts, spec_metadata = acceptance num_gens = gen_logits.shape[0] - # The draft forward's pending KV-length rewind (see _apply_kv_rewind_after_draft) runs in the kernel: every - # reader of kv_lens_cuda in the draft forward precedes it. In warmup the restore of kv_lens_cuda that - # follows overwrites it, as it would the Python rewind. - rewind = getattr(self, "_kv_rewind_amount", None) - fold = ( - getattr(self, "_kv_rewind_pending", False) - and rewind is not None - and getattr(attn_metadata, "kv_lens_cuda", None) is not None - and rewind.dtype == torch.int32 - and self._kv_rewind_bs - self._kv_rewind_nc == rewind.numel() <= num_gens - ) corrected, tokens, next_new = k3_markov_op.markov_chain( self.mapping, gen_logits.contiguous(), @@ -1272,13 +1266,7 @@ def _k3_markov_chain( accepted_tokens, num_accepted_tokens[num_contexts : num_contexts + num_gens], spec_metadata.batch_indices_cuda[num_contexts : num_contexts + num_gens], - kv_lens=attn_metadata.kv_lens_cuda if fold else None, - rewind=rewind if fold else None, - rewind_first=self._kv_rewind_nc if fold else 0, ) - if fold: - self._kv_rewind_amount = None - self._kv_rewind_pending = False self._k3_markov = (corrected, tokens, next_new, accepted_tokens, num_accepted_tokens) return corrected diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py index 9fe9802ab2c3..1916a6fa9073 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py @@ -167,7 +167,8 @@ def _k3_accept_step_applies( if ( self._ctx_block_tables is None or self._ctx_pool_windowed - or getattr(attn_metadata, "draft_kv_cache_block_offsets", None) is None + or attn_metadata.find("draft_or_none", None) is None + or attn_metadata.draft.kv_cache_block_offsets is None or getattr(attn_metadata, "kv_lens_cuda", None) is None ): return False @@ -431,11 +432,16 @@ def _on_acceptance( ) -> None: """Called with this step's acceptance when ``k3_spec_accept`` produced it (a hook for drafter families).""" - def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=None): + def _k3_accept( + self, logits, attn_metadata, spec_metadata, draft_model, draft_kv_lens, shard=None + ): """``trtllm::k3_spec_accept`` for a step ``_k3_accept_applies`` accepted: the acceptance, the block table, - the KDA replay record, kv_lens + 1 and the drafter's inputs in one launch. ``shard``: the exchange - workspace and first column when ``logits`` are this rank's vocabulary shard (``_k3_logits_shard``). - Returns (accepted tokens, num accepted, {rewind, bonus, qpos, cpos, noise}).""" + the KDA replay record, the drafter's KV lengths + 1 and the drafter's inputs in one launch. + + ``draft_kv_lens`` are the draft metadata's device KV lengths, which hold the target's at the start of the + draft loop; the kernel advances them, never the target's. ``shard``: the exchange workspace and first + column when ``logits`` are this rank's vocabulary shard (``_k3_logits_shard``). Returns (accepted tokens, + num accepted, {rewind, bonus, qpos, cpos, noise}).""" from tensorrt_llm._torch._experimental.modeling_v2.models.kimi_k3_vl.kernels.k3_spec_accept import ( op as accept_op, ) @@ -459,10 +465,10 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No pool, counter = dummy_rng embed = draft_model.draft_model_full.model.embed_tokens assert self._batch_to_slot_buffer is not None - accepted, num_acc, rewind, bonus, qpos, cpos, noise = torch.ops.trtllm.k3_spec_accept( + accepted, num_acc, _rewind, bonus, qpos, cpos, noise = torch.ops.trtllm.k3_spec_accept( logits, spec_metadata.draft_tokens.reshape(num_gens, K), - attn_metadata.draft_kv_cache_block_offsets, + attn_metadata.draft.kv_cache_block_offsets, self._ctx_pool_idx, self._ctx_block_divisor, self._ctx_block_counts, @@ -470,7 +476,7 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No mgr._kda_replay.prev_num_accepted_tokens, attn_metadata.mamba_metadata.state_indices, mgr._dummy_request_mask, - attn_metadata.kv_lens_cuda, + draft_kv_lens, self._batch_to_slot_buffer.tensor, self._ctx_len, self._max_ctx, @@ -486,7 +492,7 @@ def _k3_accept(self, logits, attn_metadata, spec_metadata, draft_model, shard=No return ( accepted, num_acc, - dict(rewind=rewind, bonus=bonus, qpos=qpos, cpos=cpos, noise=noise), + dict(bonus=bonus, qpos=qpos, cpos=cpos, noise=noise), ) @staticmethod @@ -586,208 +592,208 @@ def _forward_impl( self._execute_guided_decoder_if_present(logits) - prep = None - if fused_accept: - # Saved before the kernel updates kv_lens (warmup also snapshots kv_lens_cuda here). - self._prepare_attn_metadata_for_dflash(attn_metadata, spec_metadata) - accepted_tokens, num_accepted_tokens, prep = self._k3_accept( - logits, attn_metadata, spec_metadata, draft_model, shard - ) - else: - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata - ) + # The drafter's inputs set their lengths on the draft metadata, which leaves the target's untouched. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + prep = None + if fused_accept: + accepted_tokens, num_accepted_tokens, prep = self._k3_accept( + logits, + attn_metadata, + spec_metadata, + draft_model, + # The kernel advances the draft's KV lengths: the buffer comes from its core. + draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], + shard, + ) + else: + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + logits, attn_metadata, spec_metadata + ) - # Opt-in acceptance recording (env-gated; eager-mode measurement - # runs only). Skipped for CUDA-graph batches (capture/replay/warmup - # use synthetic requests and forbid the host sync). - if ( - self._accept_stats is not None - and num_gens > 0 - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - ): - self._accept_stats.on_accept( - spec_metadata.request_ids[num_contexts:batch_size], - num_accepted_tokens[num_contexts:batch_size].tolist(), - ) + # Opt-in acceptance recording (env-gated; eager-mode measurement + # runs only). Skipped for CUDA-graph batches (capture/replay/warmup + # use synthetic requests and forbid the host sync). + if ( + self._accept_stats is not None + and num_gens > 0 + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + ): + self._accept_stats.on_accept( + spec_metadata.request_ids[num_contexts:batch_size], + num_accepted_tokens[num_contexts:batch_size].tolist(), + ) - if fused_accept: - # The kernel recorded the KDA replay acceptance and advanced kv_lens_cuda. - self._kv_rewind_amount = prep["rewind"] - self._kv_rewind_nc = num_contexts - self._kv_rewind_bs = batch_size - self._kv_rewind_pending = True - attn_metadata.update_for_spec_dec() - else: - # Update GDN/Mamba recurrent states to the accepted token's state. - if num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): + if fused_accept: + # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft + # metadata re-derives its length-dependent state from them. + self.refresh_draft() + elif num_gens > 0 and isinstance( + attn_metadata.kv_cache_manager, MambaHybridCacheManager + ): + # Update GDN/Mamba recurrent states to the accepted token's state. attn_metadata.kv_cache_manager.update_mamba_states( attn_metadata=attn_metadata, num_accepted_tokens=num_accepted_tokens, state_indices=attn_metadata.mamba_metadata.state_indices, ) - self._prepare_attn_metadata_for_dflash(attn_metadata, spec_metadata) - self._prepare_kv_for_draft_forward( - attn_metadata, num_accepted_tokens, num_contexts, batch_size - ) - - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens + ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + prep=prep, ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, - prep=prep, - ) - if num_gens > 0: - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), - ) + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_hidden_states = self._draft_block_hidden_states( - draft_model, hidden_states_out, num_gens, block_size, K - ) - gen_logits = self._draft_block_logits( - draft_model, gen_hidden_states, attn_metadata, spec_metadata - ) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_hidden_states = self._draft_block_hidden_states( + draft_model, hidden_states_out, num_gens, block_size, K + ) + gen_logits = self._draft_block_logits( + draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata + ) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with one - # coherent path through the block (absent for plain DFlash). - if getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( - draft_model, + # DFlash 2: replace the independent per-position picks with one + # coherent path through the block (absent for plain DFlash). + if getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( + draft_model, + gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + spec_metadata, + ) + vocab_size = gen_logits.shape[-1] + + gen_draft_tokens = self.sample_draft_tokens( gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], spec_metadata, + batch_size, + num_contexts=num_contexts, ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None here -> no calibration rows collected; the - # DSpark confidence-scheduled verification MR supplies the - # real provider, see accept_stats.py). Guarded off for - # CUDA-graph batches and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], - draft_model, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - gen_draft_tokens, - ) + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None here -> no calibration rows collected; the + # DSpark confidence-scheduled verification MR supplies the + # real provider, see accept_stats.py). Guarded off for + # CUDA-graph batches and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], + draft_model, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs( + spec_metadata, num_contexts, num_gens, K, gen_vocab + ) - self._restore_attn_metadata_from_spec_dec(attn_metadata) - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._ctx_len.copy_(self._saved_ctx_len) - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False - - outputs = { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } - if shard is not None: - # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. - outputs["logits_vocab_shard"] = True - return outputs + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._ctx_len.copy_(self._saved_ctx_len) + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False + + outputs = { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } + if shard is not None: + # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. + outputs["logits_vocab_shard"] = True + return outputs def _draft_block_hidden_states( self, @@ -1045,10 +1051,12 @@ def prepare_1st_drafter_inputs( # Update seq_lens for gen requests to K+1 (the number of tokens # the target forward actually processed). - attn_metadata._seq_lens_cuda[num_contexts : num_contexts + num_gens] = ( - total_tokens_per_req + attn_metadata.set_seq_lens( + slice(num_contexts, num_contexts + num_gens), + host=total_tokens_per_req, + device=total_tokens_per_req, ) - attn_metadata._seq_lens[num_contexts : num_contexts + num_gens] = total_tokens_per_req + self.refresh_draft() else: noise_embedding = hidden_states.new_empty(0, 0, hidden_dim) query_positions = torch.empty(0, 0, dtype=torch.long, device="cuda") @@ -1221,10 +1229,9 @@ def sample_and_accept_draft_tokens(self, logits, attn_metadata, spec_metadata): def _on_acceptance( self, accepted_tokens, num_accepted_tokens, attn_metadata, spec_metadata ) -> None: - """Keeps this step's acceptance for ``k3_markov``'s next_new_tokens (and the metadata whose KV lengths it - rewinds).""" + """Keeps this step's acceptance for ``k3_markov``'s next_new_tokens.""" self._k3_acceptance = ( - accepted_tokens, num_accepted_tokens, attn_metadata.num_contexts, spec_metadata, attn_metadata + accepted_tokens, num_accepted_tokens, attn_metadata.num_contexts, spec_metadata ) # fmt: skip def _k3_markov_chain( @@ -1247,21 +1254,8 @@ def _k3_markov_chain( acceptance = getattr(self, "_k3_acceptance", None) if acceptance is None: raise RuntimeError("DSpark Markov head: k3_markov needs this step's acceptance first") - accepted_tokens, num_accepted_tokens, num_contexts, spec_metadata, attn_metadata = ( - acceptance - ) + accepted_tokens, num_accepted_tokens, num_contexts, spec_metadata = acceptance num_gens = gen_logits.shape[0] - # The draft forward's pending KV-length rewind (see _apply_kv_rewind_after_draft) runs in the kernel: every - # reader of kv_lens_cuda in the draft forward precedes it. In warmup the restore of kv_lens_cuda that - # follows overwrites it, as it would the Python rewind. - rewind = getattr(self, "_kv_rewind_amount", None) - fold = ( - getattr(self, "_kv_rewind_pending", False) - and rewind is not None - and getattr(attn_metadata, "kv_lens_cuda", None) is not None - and rewind.dtype == torch.int32 - and self._kv_rewind_bs - self._kv_rewind_nc == rewind.numel() <= num_gens - ) corrected, tokens, next_new = k3_markov_op.markov_chain( self.mapping, gen_logits.contiguous(), @@ -1272,13 +1266,7 @@ def _k3_markov_chain( accepted_tokens, num_accepted_tokens[num_contexts : num_contexts + num_gens], spec_metadata.batch_indices_cuda[num_contexts : num_contexts + num_gens], - kv_lens=attn_metadata.kv_lens_cuda if fold else None, - rewind=rewind if fold else None, - rewind_first=self._kv_rewind_nc if fold else 0, ) - if fold: - self._kv_rewind_amount = None - self._kv_rewind_pending = False self._k3_markov = (corrected, tokens, next_new, accepted_tokens, num_accepted_tokens) return corrected diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py index c0bf5cd12b17..8234090bb0f0 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/modeling.py @@ -2223,7 +2223,9 @@ def step_args(self, md) -> dict: num_tokens=md.num_tokens, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), metadata=md, ) @@ -2514,11 +2516,13 @@ def step_args(self, md) -> dict: num_seqs=md.num_seqs, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), msa_decode_span=md.msa_decode_span, msa_block_table=md.msa_block_table, msa_seq_lens_cuda=md.msa_seq_lens_cuda, - msa_short_ok=md._msa_short_ok, + msa_short_ok=md.msa_short_table_staged, msa_short_table=md.msa_short_table, msa_proxy_max_score_view=md.msa_proxy_max_score_view, msa_worst_case_max_k_tiles=md.msa_worst_case_max_k_tiles, @@ -3721,46 +3725,41 @@ def _forward_impl( batch_size=batch_size, ) else: - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) - - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) - # can put attn_metadata back into a state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # The draft forwards run on the draft metadata, which leaves the target's untouched; without one, + # the target's state is saved and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - next_draft_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) + # can put attn_metadata back into a state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + + next_draft_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + ) # prepare next new tokens to support overlap scheduler if self._m3_next_new_tokens is not None: @@ -4406,18 +4405,15 @@ def _m3_fused_draft_loop( ids, pos, hidden = ids_next, pos_next, to_save if i == 0: # The draft step set the device sequence lengths to 1 and rolled the KV lengths back. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - has_kv_cache = md.kv_cache_manager is not None - if has_kv_cache and hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - if has_kv_cache: - md.num_contexts = 0 + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + if md.kv_cache_manager is not None: + # The context requests continue as one-token generation rows. + md.as_generation_rows() self._prepare_flash_mla_generation_layout(md, 0, batch_size) - md.update_for_spec_dec() + self.refresh_draft() md.use_spec_decoding = False else: - md.update_for_spec_dec() + self.refresh_draft() self._m3_next_new_tokens = next_new return draft_tokens diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py index 1e8c520ca9d6..daba3bcf2936 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/modeling.py @@ -1936,7 +1936,9 @@ def step_args(self, md) -> dict: num_tokens=md.num_tokens, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), metadata=md, ) @@ -2227,11 +2229,13 @@ def step_args(self, md) -> dict: num_seqs=md.num_seqs, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), msa_decode_span=md.msa_decode_span, msa_block_table=md.msa_block_table, msa_seq_lens_cuda=md.msa_seq_lens_cuda, - msa_short_ok=md._msa_short_ok, + msa_short_ok=md.msa_short_table_staged, msa_short_table=md.msa_short_table, msa_proxy_max_score_view=md.msa_proxy_max_score_view, msa_worst_case_max_k_tiles=md.msa_worst_case_max_k_tiles, @@ -3441,46 +3445,41 @@ def _forward_impl( batch_size=batch_size, ) else: - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) - - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) - # can put attn_metadata back into a state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # The draft forwards run on the draft metadata, which leaves the target's untouched; without one, + # the target's state is saved and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - next_draft_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) + # can put attn_metadata back into a state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + + next_draft_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + ) # prepare next new tokens to support overlap scheduler if self._m3_next_new_tokens is not None: @@ -4075,18 +4074,15 @@ def _m3_fused_draft_loop( ids, pos, hidden = ids_next, pos_next, to_save if i == 0: # The draft step set the device sequence lengths to 1 and rolled the KV lengths back. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - has_kv_cache = md.kv_cache_manager is not None - if has_kv_cache and hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - if has_kv_cache: - md.num_contexts = 0 + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + if md.kv_cache_manager is not None: + # The context requests continue as one-token generation rows. + md.as_generation_rows() self._prepare_flash_mla_generation_layout(md, 0, batch_size) - md.update_for_spec_dec() + self.refresh_draft() md.use_spec_decoding = False else: - md.update_for_spec_dec() + self.refresh_draft() self._m3_next_new_tokens = next_new return draft_tokens diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py index 5e4b8b9532f6..772b311a50b5 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py @@ -316,7 +316,6 @@ from .ops.spec.m3_draft_step import m3_draft_step from .ops.spec.m3_sampled_accept import m3_sampled_accept from .ops.spec.m3_shard_accept import m3_shard_accept -from .ops.spec.m3_spec_offsets import m3_spec_offsets from .ops.spec.m3_step_prologue import m3_step_prologue _SM = (10, 7) @@ -513,7 +512,7 @@ class DecodePlan: # block tables, positions, cache slots, the sampled-accept pre-wait parameters) before their grid-dependency wait rely # on every writer of them being a host-to-device copy before the replay or a PLAIN launch earlier in the step: # msa_kv_update, the step-input prologue (Triton scatter or the C++ op), the spec fold tail (Triton or C++), the pack -# scatter and any step-head op (the spec step correction, STEP_SPEC_OFFSETS_ONE_LAUNCH). Never give one of them a PDL +# scatter. Never give one of them a PDL # attribute with an early trigger. PRODUCER_PREWAIT_STEP_INPUTS = True @@ -2904,7 +2903,7 @@ def _prefill_attention_boundary( position_ids[..., :live] if position_ids is not None else None, md.kv_cache_manager, md.msa_idx_k_cache, - md.msa_out_cache_loc, + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc, ) if horizontal is None: raise RuntimeError( @@ -2974,7 +2973,9 @@ def step_args(self, md) -> dict: num_tokens=md.num_tokens, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), metadata=md, ) @@ -3302,11 +3303,13 @@ def step_args(self, md) -> dict: max_seq_len=md.max_seq_len, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), msa_decode_span=md.msa_decode_span, msa_block_table=md.msa_block_table, msa_seq_lens_cuda=md.msa_seq_lens_cuda, - msa_short_ok=md._msa_short_ok, + msa_short_ok=md.msa_short_table_staged, msa_short_table=md.msa_short_table, msa_proxy_max_score_view=md.msa_proxy_max_score_view, msa_worst_case_max_k_tiles=md.msa_worst_case_max_k_tiles, @@ -5077,50 +5080,46 @@ def _forward_impl( batch_size=batch_size, ) else: - # Retain the pre-draft tensor before metadata is replaced, as in the parent worker. + # The step-0 sequence lengths of the target, which the draft loop never changes when it runs on + # the draft metadata. pre_draft_seq_lens = attn_metadata._seq_lens_cuda - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) + # The draft forwards run on the draft metadata, which leaves the target's untouched; without one, + # the target's state is saved and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) - # can put attn_metadata back into a state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) + # can put attn_metadata back into a state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - next_draft_tokens, next_new_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - accepted_tokens, - pre_draft_seq_lens, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + next_draft_tokens, next_new_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + accepted_tokens, + pre_draft_seq_lens, + ) # prepare next new tokens to support overlap scheduler if self._m3_next_new_tokens is not None: @@ -6002,18 +6001,15 @@ def _m3_fused_draft_loop( ids, pos, hidden = ids_next, pos_next, to_save if i == 0: # The draft step set the device sequence lengths to 1 and rolled the KV lengths back. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - has_kv_cache = md.kv_cache_manager is not None - if has_kv_cache and hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - if has_kv_cache: - md.num_contexts = 0 + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + if md.kv_cache_manager is not None: + # The context requests continue as one-token generation rows. + md.as_generation_rows() self._prepare_flash_mla_generation_layout(md, 0, batch_size) - md.update_for_spec_dec() + self.refresh_draft() md.use_spec_decoding = False else: - md.update_for_spec_dec() + self.refresh_draft() self._m3_next_new_tokens = next_new return draft_tokens @@ -6040,11 +6036,6 @@ def _runtime_hooks(spec_fold: Optional[M3SpecStepFold]) -> RuntimeHooks: batches stage their Philox seed/offset uploads, and the accepted-draft counts, into the same pack; the sampler's slot ids stay in a device buffer re-uploaded only when they change. * ``skip_unchanged_h2d``: steady decode re-uploads slot indices, query lengths and page tables that rarely change. - * ``kv_lens_update_once``: a speculative overlap step re-derives the MSA KV-length metadata once, after the - forward's KV-length correction (the first pass is rewritten in full by the second). - * ``spec_offsets_op``: the overlap scheduler's speculative step correction (position_ids += offsets, kv_lens += - offsets) as this target's one plain launch (``m3_spec_offsets``) instead of two ATen adds at the head of the - decode graph: the same int32 adds. * ``loop_on_execution_stream``: the overlap loop's own stream work and the forward on one stream, so step N+1's inputs wait on graph N in program order rather than through two cross-stream event hops. * ``merge_tuned_tactics_across_ranks``: tensor-parallel peers capture the same GEMM tactics (rank-local profiling @@ -6071,8 +6062,6 @@ def _runtime_hooks(spec_fold: Optional[M3SpecStepFold]) -> RuntimeHooks: pack_early_copy=True, pack_prologue_op=m3_step_prologue, skip_unchanged_h2d=True, - kv_lens_update_once=True, - spec_offsets_op=m3_spec_offsets, loop_on_execution_stream=True, merge_tuned_tactics_across_ranks=True, skip_greedy_rng_upload=True, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/ops/spec/m3_spec_offsets.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/ops/spec/m3_spec_offsets.py deleted file mode 100644 index 6a766b8dc0ce..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/ops/spec/m3_spec_offsets.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""The overlap scheduler's speculative step correction as one launch (``RuntimeHooks.spec_offsets_op``).""" - -from __future__ import annotations - -import torch - -import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* -from tensorrt_llm._torch._experimental.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm.logger import logger - -# The engagement line is logged once per process (module state: __call__ reads no instance state). -_ENGAGED: list = [] - - -class M3SpecOffsets(OpWrapper): - """``positions += position_offsets`` and ``kv_lens += kv_lens_offsets`` (int32, contiguous 1-D views, equal lengths - per pair) in one plain launch: trtllm::minimax_m3_sm107_tp2_spec_offsets (minimaxM3Sm107/specOffsets.cu), the - same wrapping int32 adds the two ATen ops do. ``__call__`` returns False, having written nothing, for other - layouts (the caller runs the ATen adds).""" - - ARCHS = frozenset({Arch.SM_107}) - - CELLS: tuple[Cell, ...] = ( - Cell(why="1 request, 4 tokens (draft length 3)", spec=dict(tokens=4, seqs=1)), - Cell(why="8 requests, 32 tokens", spec=dict(tokens=32, seqs=8)), - Cell(why="wrap-around values", spec=dict(tokens=8, seqs=2, wrap=True)), - ) - - note = "Integer adds: the reference (the two in-place ATen adds) is exact." - - def __call__(self, pos, pos_off, kv, kv_off) -> bool: - ts = (pos, pos_off, kv, kv_off) - if ( - any( - t.dtype != torch.int32 or t.dim() != 1 or t.stride(0) != 1 or not t.is_cuda - for t in ts - ) - or pos.numel() != pos_off.numel() - or kv.numel() != kv_off.numel() - ): - return False - torch.ops.trtllm.minimax_m3_sm107_tp2_spec_offsets(pos, pos_off, kv, kv_off) - if not _ENGAGED: - _ENGAGED.append(True) - logger.info("[M3 host] spec step correction: one launch taken (spec_offsets_op)") - return True - - def reference(self, pos, pos_off, kv, kv_off) -> None: - pos += pos_off - kv += kv_off - - def is_valid(self, pos, pos_off, kv, kv_off) -> None: - assert pos.numel() == pos_off.numel() and kv.numel() == kv_off.numel(), ( - "equal lengths per pair" - ) - - -m3_spec_offsets = M3SpecOffsets() diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py index 8745cd6e4d95..def7252f4e9e 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/modeling.py @@ -296,7 +296,6 @@ from .ops.spec.m3_draft_step import m3_draft_step from .ops.spec.m3_force_vec_req import m3_force_vec_req from .ops.spec.m3_shard_accept import m3_shard_accept -from .ops.spec.m3_spec_offsets import m3_spec_offsets from .ops.spec.m3_step_prologue import m3_step_prologue from .ops.triton_sparse_decode import MERGE_MXFP8, MERGE_MXFP8_MIN_TOKENS @@ -1064,7 +1063,7 @@ def _long_select_dsl_widths(spec: Mapping[str, Any]) -> Iterable[Key]: # block tables, positions, cache slots, the sampled-accept pre-wait parameters) before their grid-dependency wait rely # on every writer of them being a host-to-device copy before the replay or a PLAIN launch earlier in the step: # msa_kv_update, the step-input prologue (Triton scatter or the C++ op), the spec fold tail (Triton or C++), the pack -# scatter and any step-head op (the spec step correction, STEP_SPEC_OFFSETS_ONE_LAUNCH). Never give one of them a PDL +# scatter. Never give one of them a PDL # attribute with an early trigger. STEP_SKIP_UNCHANGED_H2D = True # A changed packed 2-D metadata upload stages only each row's changed column span. On a step where a request crosses @@ -1072,16 +1071,6 @@ def _long_select_dsl_widths(spec: Mapping[str, Any]) -> Iterable[Key]: # overflowing the 16 KB in-graph window into an eager upload and scatter between the step graphs (57-69 us at the # ~1 in 19 block-crossing steps; TTFT inter-graph scan). Kill: False. STEP_H2D_CHANGED_SPAN = True -# A speculative overlap step re-derives the MSA KV-length metadata once, after the forward's KV-length correction, -# instead of also before it (RuntimeHooks.kv_lens_update_once): the first pass (a fill, the out_cache_loc tail fill and -# an msa_kv_update at the head of the decode graph, 2.13 us at TP4) is rewritten in full by the second. Kill: False. -STEP_KV_LENS_UPDATE_ONCE = True -# The overlap scheduler's speculative step correction (position_ids += offsets, kv_lens += offsets) runs as this -# target's one plain launch (RuntimeHooks.spec_offsets_op, ops/spec/m3_spec_offsets) instead of two ATen adds at the -# head of the decode graph: the same int32 adds. TP8 graph bench (the work is per token, not per rank; 64 steps -# per graph, a predecessor kernel each): -1.27 us/step at 4 tokens / 1 request, -0.72..-0.78 at 8-32 tokens. -# Kill: False. -STEP_SPEC_OFFSETS_ONE_LAUNCH = True STEP_LOOP_ON_EXECUTION_STREAM = True STEP_MERGE_TUNED_TACTICS = True STEP_SKIP_GREEDY_RNG_UPLOAD = True @@ -2702,7 +2691,9 @@ def step_args(self, md) -> dict: num_tokens=md.num_tokens, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), metadata=md, ) @@ -3004,11 +2995,13 @@ def step_args(self, md) -> dict: num_seqs=md.num_seqs, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), msa_decode_span=md.msa_decode_span, msa_block_table=md.msa_block_table, msa_seq_lens_cuda=md.msa_seq_lens_cuda, - msa_short_ok=md._msa_short_ok, + msa_short_ok=md.msa_short_table_staged, msa_short_table=md.msa_short_table, msa_proxy_max_score_view=md.msa_proxy_max_score_view, msa_worst_case_max_k_tiles=md.msa_worst_case_max_k_tiles, @@ -4602,50 +4595,46 @@ def _forward_impl( batch_size=batch_size, ) else: - # Retain the pre-draft tensor before metadata is replaced, as in the parent worker. + # The step-0 sequence lengths of the target, which the draft loop never changes when it runs on + # the draft metadata. pre_draft_seq_lens = attn_metadata._seq_lens_cuda - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) + # The draft forwards run on the draft metadata, which leaves the target's untouched; without one, + # the target's state is saved and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) - # can put attn_metadata back into a state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) + # can put attn_metadata back into a state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - next_draft_tokens, next_new_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - accepted_tokens, - pre_draft_seq_lens, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + next_draft_tokens, next_new_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + accepted_tokens, + pre_draft_seq_lens, + ) # prepare next new tokens to support overlap scheduler if self._m3_next_new_tokens is not None: @@ -5482,18 +5471,15 @@ def _m3_fused_draft_loop( ids, pos, hidden = ids_next, pos_next, to_save if i == 0: # The draft step set the device sequence lengths to 1 and rolled the KV lengths back. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - has_kv_cache = md.kv_cache_manager is not None - if has_kv_cache and hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - if has_kv_cache: - md.num_contexts = 0 + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + if md.kv_cache_manager is not None: + # The context requests continue as one-token generation rows. + md.as_generation_rows() self._prepare_flash_mla_generation_layout(md, 0, batch_size) - md.update_for_spec_dec() + self.refresh_draft() md.use_spec_decoding = False else: - md.update_for_spec_dec() + self.refresh_draft() self._m3_next_new_tokens = next_new return draft_tokens @@ -5542,8 +5528,6 @@ def _runtime_hooks(spec_fold: Optional[M3SpecStepFold]) -> RuntimeHooks: pack_prologue_op=m3_step_prologue if STEP_PACK_CPP_PROLOGUE else None, skip_unchanged_h2d=STEP_SKIP_UNCHANGED_H2D, h2d_changed_span=STEP_H2D_CHANGED_SPAN, - kv_lens_update_once=STEP_KV_LENS_UPDATE_ONCE, - spec_offsets_op=m3_spec_offsets if STEP_SPEC_OFFSETS_ONE_LAUNCH else None, loop_on_execution_stream=STEP_LOOP_ON_EXECUTION_STREAM, merge_tuned_tactics_across_ranks=STEP_MERGE_TUNED_TACTICS, skip_greedy_rng_upload=STEP_SKIP_GREEDY_RNG_UPLOAD, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/ops/spec/m3_spec_offsets.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/ops/spec/m3_spec_offsets.py deleted file mode 100644 index 3094e2e9965a..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/ops/spec/m3_spec_offsets.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""The overlap scheduler's speculative step correction as one launch (``RuntimeHooks.spec_offsets_op``).""" - -from __future__ import annotations - -import torch - -import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* -from tensorrt_llm._torch._experimental.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm.logger import logger - -# The engagement line is logged once per process (module state: __call__ reads no instance state). -_ENGAGED: list = [] - - -class M3SpecOffsets(OpWrapper): - """``positions += position_offsets`` and ``kv_lens += kv_lens_offsets`` (int32, contiguous 1-D views, equal lengths - per pair) in one plain launch: trtllm::minimax_m3_sm107_tp4_spec_offsets (minimaxM3Sm107/specOffsets.cu), the - same wrapping int32 adds the two ATen ops do. ``__call__`` returns False, having written nothing, for other - layouts (the caller runs the ATen adds).""" - - ARCHS = frozenset({Arch.SM_107}) - - CELLS: tuple[Cell, ...] = ( - Cell(why="1 request, 4 tokens (draft length 3)", spec=dict(tokens=4, seqs=1)), - Cell(why="8 requests, 32 tokens", spec=dict(tokens=32, seqs=8)), - Cell(why="wrap-around values", spec=dict(tokens=8, seqs=2, wrap=True)), - ) - - note = "Integer adds: the reference (the two in-place ATen adds) is exact." - - def __call__(self, pos, pos_off, kv, kv_off) -> bool: - ts = (pos, pos_off, kv, kv_off) - if ( - any( - t.dtype != torch.int32 or t.dim() != 1 or t.stride(0) != 1 or not t.is_cuda - for t in ts - ) - or pos.numel() != pos_off.numel() - or kv.numel() != kv_off.numel() - ): - return False - torch.ops.trtllm.minimax_m3_sm107_tp4_spec_offsets(pos, pos_off, kv, kv_off) - if not _ENGAGED: - _ENGAGED.append(True) - logger.info("[M3 host] spec step correction: one launch taken (spec_offsets_op)") - return True - - def reference(self, pos, pos_off, kv, kv_off) -> None: - pos += pos_off - kv += kv_off - - def is_valid(self, pos, pos_off, kv, kv_off) -> None: - assert pos.numel() == pos_off.numel() and kv.numel() == kv_off.numel(), ( - "equal lengths per pair" - ) - - -m3_spec_offsets = M3SpecOffsets() diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py index f5e12222b841..a0063316edc3 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/modeling.py @@ -307,7 +307,6 @@ from .ops.spec.m3_draft_step import m3_draft_step from .ops.spec.m3_sampled_accept import m3_sampled_accept from .ops.spec.m3_shard_accept import m3_shard_accept -from .ops.spec.m3_spec_offsets import m3_spec_offsets from .ops.spec.m3_step_prologue import m3_step_prologue _SM = (10, 7) @@ -446,9 +445,7 @@ class DecodePlan: # value) runs the shared minimax_m3_producer_pw at 8 query heads, which reads them after its wait. # Step-input writers added since: the step-input prologue's C++ op (STEP_PACK_CPP_PROLOGUE) and the C++ spec fold tail # (STEP_FOLD_TAIL_CPP) are plain launches like the Triton kernels they replace (their tests assert it on the captured -# graph); STEP_KV_LENS_UPDATE_ONCE only drops an earlier writer whose outputs the surviving msa_kv_update rewrites. -# The spec step correction (STEP_SPEC_OFFSETS_ONE_LAUNCH: position_ids, kv_lens) is a plain launch too (its test -# asserts it). +# graph). PRODUCER_PREWAIT_STEP_INPUTS = True # #117: on a decode step of at most 8 rows the sparse layer's QKV + index GEMV runs the producer in its owner epilogue # (ops/mx_gemv m3_mx_gemv_producer, m3_layer.phase_producer.producer_head): one launch and one PDL edge fewer per @@ -2765,7 +2762,9 @@ def step_args(self, md) -> dict: num_tokens=md.num_tokens, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), metadata=md, ) @@ -3075,11 +3074,13 @@ def step_args(self, md) -> dict: max_seq_len=md.max_seq_len, kv_cache_manager=md.kv_cache_manager, msa_idx_k_cache=md.msa_idx_k_cache, - msa_out_cache_loc=md.msa_out_cache_loc, + msa_out_cache_loc=( + md.msa_write_slots() if hasattr(md, "msa_write_slots") else md.msa_out_cache_loc + ), msa_decode_span=md.msa_decode_span, msa_block_table=md.msa_block_table, msa_seq_lens_cuda=md.msa_seq_lens_cuda, - msa_short_ok=md._msa_short_ok, + msa_short_ok=md.msa_short_table_staged, msa_short_table=md.msa_short_table, msa_proxy_max_score_view=md.msa_proxy_max_score_view, msa_worst_case_max_k_tiles=md.msa_worst_case_max_k_tiles, @@ -4776,50 +4777,46 @@ def _forward_impl( batch_size=batch_size, ) else: - # Retain the pre-draft tensor before metadata is replaced, as in the parent worker. + # The step-0 sequence lengths of the target, which the draft loop never changes when it runs on + # the draft metadata. pre_draft_seq_lens = attn_metadata._seq_lens_cuda - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) + # The draft forwards run on the draft metadata, which leaves the target's untouched; without one, + # the target's state is saved and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) - # can put attn_metadata back into a state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here so the post-loop restore (below) + # can put attn_metadata back into a state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - next_draft_tokens, next_new_tokens = self._forward_draft_loop( - inputs, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - num_accepted_tokens, - original_all_rank_num_tokens, - resource_manager, - accepted_tokens, - pre_draft_seq_lens, - ) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + next_draft_tokens, next_new_tokens = self._forward_draft_loop( + inputs, + draft_attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + num_accepted_tokens, + original_all_rank_num_tokens, + resource_manager, + accepted_tokens, + pre_draft_seq_lens, + ) self._m3_join_pending() # prepare next new tokens to support overlap scheduler @@ -5685,18 +5682,15 @@ def _m3_fused_draft_loop( ids, pos, hidden = ids_next, pos_next, to_save if i == 0: # The draft step set the device sequence lengths to 1 and rolled the KV lengths back. - md._seq_lens[:batch_size].fill_(1) - md.on_update() - has_kv_cache = md.kv_cache_manager is not None - if has_kv_cache and hasattr(md, "host_request_types"): - md.host_request_types[: md.num_contexts].fill_(1) - if has_kv_cache: - md.num_contexts = 0 + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + if md.kv_cache_manager is not None: + # The context requests continue as one-token generation rows. + md.as_generation_rows() self._prepare_flash_mla_generation_layout(md, 0, batch_size) - md.update_for_spec_dec() + self.refresh_draft() md.use_spec_decoding = False else: - md.update_for_spec_dec() + self.refresh_draft() self._m3_next_new_tokens = next_new return draft_tokens @@ -5730,11 +5724,6 @@ def _runtime_hooks(spec_fold: Optional[M3SpecStepFold]) -> RuntimeHooks: where a request crosses into a new page the MSA block table changed by one entry, but the whole padded table (32 KB a request) was staged, overflowing the 16 KB in-graph window into an eager upload and scatter between the step graphs (TP4 57-69 us at the ~1 in 19 block-crossing steps). - * ``kv_lens_update_once``: a speculative overlap step re-derives the MSA KV-length metadata once, after the - forward's KV-length correction (the first pass is rewritten in full by the second). - * ``spec_offsets_op``: the overlap scheduler's speculative step correction (position_ids += offsets, kv_lens += - offsets) as this target's one plain launch (``m3_spec_offsets``) instead of two ATen adds at the head of the - decode graph: the same int32 adds. * ``loop_on_execution_stream``: the overlap loop's own stream work and the forward on one stream, so step N+1's inputs wait on graph N in program order rather than through two cross-stream event hops. * ``merge_tuned_tactics_across_ranks``: tensor-parallel peers capture the same GEMM tactics (rank-local profiling @@ -5762,8 +5751,6 @@ def _runtime_hooks(spec_fold: Optional[M3SpecStepFold]) -> RuntimeHooks: pack_step_epoch=True, pack_prologue_op=m3_step_prologue, skip_unchanged_h2d=True, - kv_lens_update_once=True, - spec_offsets_op=m3_spec_offsets, h2d_changed_span=True, loop_on_execution_stream=True, merge_tuned_tactics_across_ranks=True, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/ops/spec/m3_spec_offsets.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/ops/spec/m3_spec_offsets.py deleted file mode 100644 index a0602ced8764..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/ops/spec/m3_spec_offsets.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""The overlap scheduler's speculative step correction as one launch (``RuntimeHooks.spec_offsets_op``).""" - -from __future__ import annotations - -import torch - -import tensorrt_llm._torch.custom_ops # noqa: F401 — registers torch.ops.trtllm.* -from tensorrt_llm._torch._experimental.modeling_v2.catalog._op import Arch, Cell, OpWrapper -from tensorrt_llm.logger import logger - -# The engagement line is logged once per process (module state: __call__ reads no instance state). -_ENGAGED: list = [] - - -class M3SpecOffsets(OpWrapper): - """``positions += position_offsets`` and ``kv_lens += kv_lens_offsets`` (int32, contiguous 1-D views, equal lengths - per pair) in one plain launch: trtllm::minimax_m3_sm107_tp8_spec_offsets (minimaxM3Sm107/specOffsets.cu), the - same wrapping int32 adds the two ATen ops do. ``__call__`` returns False, having written nothing, for other - layouts (the caller runs the ATen adds).""" - - ARCHS = frozenset({Arch.SM_107}) - - CELLS: tuple[Cell, ...] = ( - Cell(why="1 request, 4 tokens (draft length 3)", spec=dict(tokens=4, seqs=1)), - Cell(why="8 requests, 32 tokens", spec=dict(tokens=32, seqs=8)), - Cell(why="wrap-around values", spec=dict(tokens=8, seqs=2, wrap=True)), - ) - - note = "Integer adds: the reference (the two in-place ATen adds) is exact." - - def __call__(self, pos, pos_off, kv, kv_off) -> bool: - ts = (pos, pos_off, kv, kv_off) - if ( - any( - t.dtype != torch.int32 or t.dim() != 1 or t.stride(0) != 1 or not t.is_cuda - for t in ts - ) - or pos.numel() != pos_off.numel() - or kv.numel() != kv_off.numel() - ): - return False - torch.ops.trtllm.minimax_m3_sm107_tp8_spec_offsets(pos, pos_off, kv, kv_off) - if not _ENGAGED: - _ENGAGED.append(True) - logger.info("[M3 host] spec step correction: one launch taken (spec_offsets_op)") - return True - - def reference(self, pos, pos_off, kv, kv_off) -> None: - pos += pos_off - kv += kv_off - - def is_valid(self, pos, pos_off, kv, kv_off) -> None: - assert pos.numel() == pos_off.numel() and kv.numel() == kv_off.numel(), ( - "equal lengths per pair" - ) - - -m3_spec_offsets = M3SpecOffsets() diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index 2807813eae62..5ad6f9c30e14 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -1790,8 +1790,8 @@ def _bind_draft_cache_manager(self, md) -> None: """Bind the draft layer to the manager its draft step sees: its own index plane, that manager's position plane and its paged K/V. - Under attention TP that is the separate draft KV cache the worker swaps in for the - draft loop, whose pools hold the draft layer's pages and none of the trunk's; + Under attention TP that is the separate draft KV cache of the draft metadata the + draft loop runs on, whose pools hold the draft layer's pages and none of the trunk's; otherwise the trunk's manager, which maps the draft layer as well. Keyed on the manager like `_bind_cache_manager`, and never first bound under capture: the runner runs every captured batch shape eagerly first.""" @@ -1838,9 +1838,9 @@ def _bind_draft_cache_manager(self, md) -> None: def _draft_step_bind(self, md) -> tuple[_DraftBound, int]: """The draft layer's bind for this draft step's manager, and the layer's pool offset - in it (what `thop_attention` addresses), checked on every call: the worker swaps the - manager and the page tables in for the draft loop, and a page table still decoding - the trunk's offsets would read and write another pool's pages.""" + in it (what `thop_attention` addresses), checked on every call: with a separate draft + KV cache the draft loop runs on a draft metadata over that manager, whose + `qsa_block_table` decodes the draft manager's offsets.""" cm = md.kv_cache_manager assert cm is not None, "a paged KV cache is required" offsets = cm.layer_offsets @@ -1857,13 +1857,6 @@ def _draft_step_bind(self, md) -> tuple[_DraftBound, int]: ) if self._draft_bound is None or cm is not self._draft_bound.cm: self._bind_draft_cache_manager(md) - assert cm is self._bound.cm or md.qsa_block_table is getattr( - md, "qsa_draft_block_table", None - ), ( - "the draft step is bound to a separate draft KV cache manager but qsa_block_table " - "is not that manager's table: the engine's QSA metadata did not swap its page " - "table for the draft forward" - ) return self._draft_bound, layer_idx # -------------------------------------------------------------- forward --- diff --git a/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md index af61de7c0ccf..a8a257da1eea 100644 --- a/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md @@ -354,17 +354,190 @@ metadata and cache behavior. ### 3.1 Metadata families -All backend metadata types inherit from `AttentionMetadata`. The base contract -includes sequence-length and request-level state, KV-cache manager and -parameters, runtime feature flags, and optional CUDA-graph buffer management. -Sparse metadata subtypes consume only `SparseMetadataParams`, not -backend-owned `SparseParams`. +All backend metadata types inherit from `AttentionMetadata`, a container of +extensions with no state of its own: it holds the extension registry and the +lifecycle entry points (`prepare` and CUDA-graph copies). Its constructor routes every keyword argument to the extension that +takes it (`init_fields`); an argument for an extension that is not attached, +or that no extension takes, is an error. + +The state lives in the family's core, an `AttentionCoreMetadata` +(`backends/core.py`) extension that is always attached first. The base core +holds the construction configuration (capacities, mapping, KV cache manager, +runtime features, CUDA-graph flag and buffer pool, cache indirection) and the +batch layout and inputs (query/KV lengths, request and token counts, request +IDs, prompt lengths, offsets, KV-cache parameters, beam width, attention DP +token counts); `TrtllmCoreMetadata`, `FlashInferCoreMetadata` and +`VanillaCoreMetadata` add the family's buffers and planner state, allocate them +in their constructor and derive them per step in `derive()`. A family picks its core +with `core_metadata_type`, and the constructor routes the core's arguments to +it. Because the metadata forwards the core's declared fields, `metadata.seq_lens` +and `metadata.kv_lens_cuda` read the core. `copy.copy` of a metadata copies its +core, so a view owns its batch state. + +Everything a feature, a sparse algorithm or a model adds on top of a core +family is a `MetadataExtension` (`backends/extensions.py`) attached to the +metadata, not a metadata subclass. Callers reach one with `metadata.ext(T)` +(raises when absent) or `metadata.find_ext(T)` (returns None). The metadata +drives every extension through two required hooks: the constructor on +creation and on every CUDA-graph, cross-attention or draft copy, and +`prepare()` once per step after the core derived. The constructor +`__init__(metadata, *, source=None, **config)` calls `super().__init__` first, +which sets every creation argument the class declares in `init_fields` (a map +from name to default) from the routed arguments or the default; it then sets +the extension's other state and allocates its persistent buffers. State every +copy shares with its source (declared in `shared`) comes from +`self.shared_or(name, allocate)`. There is no constructor that skips +allocation: an extension that is not attached is absent. Two members are +opt-in and called only by `DraftMetadata` on a draft's extensions: +`adopt_target_state(source)` and `refresh_from_draft_lengths()`, which +refreshes the state derived from the draft's lengths after the draft loop +changed them; an override calls `super()`. Length-derived state an extension +computes lazily is invalidated by the extension itself in `prepare()` and +`refresh_from_draft_lengths()`. A model declares the extensions it needs and +their creation arguments through `RuntimeHooks.metadata_extensions` / +`metadata_config` (for example MiniMax-M3's `msa_policy`); its per-step state +lives in a model extension or on the model, never in a module-level table +keyed by the metadata. Extensions come from three places: + +- the core family's own configuration (`default_extension_types()`), such as + `HelixMetadata`, `DraftMetadata`, `Fp4MlaState`, `SpecDecodingMetadata` + and `RecurrentMetadata`; +- the backend's `metadata_extensions`, such as a sparse algorithm's + `SparseAlgorithmExtension` (at most one per metadata); and +- the caller's `extension_types`, such as `FlashMlaMetadata`. + +The engine describes each step with a `BatchInputs` (sequence lengths, +context count, request IDs, prompt lengths, KV-cache parameters, +beam width, optional packed-token offsets, `kv_lens_offsets`, +`position_id_offsets`, and `cross` for the cross-attention sub-metadata) and +calls +`metadata.prepare(inputs, context_requests=...)`. Every forward of a metadata +follows its own `prepare()`; CUDA-graph warmup and capture prepare again +before each forward (`CUDAGraphRunner.capture(..., prepare_each_forward=True)`). +`kv_lens_offsets` carries the overlap scheduler's KV-length correction for +speculative decoding (the previous step's accepted draft tokens, known only on +device): the TRTLLM core adds it to the device KV lengths in `prepare()`, so +the device lengths may be shorter than the host ones, and each extension +derives its length-dependent device state from the device lengths there. +`position_id_offsets` carries the same correction per generation token for +the positions; `HelixMetadata` applies it to its global positions and derives +the rank-local KV lengths of speculative verify groups on device in its +`prepare()`, which supersedes the `kv_lens_offsets` correction. Extensions +prepared after Helix (DSA, FP4 MLA) derive from those lengths. + +Write access follows field ownership. Lengths and counts belong to the core; code that changes them after `prepare()` uses the +core's methods, which keep the graph-stable buffers: `adjust_kv_lens()` / +`set_kv_lens()` for the device KV lengths, `adjust_seq_lens()` / +`set_seq_lens()` / `bind_lengths()` / `copy_lengths_from()` for the query +lengths and counts, and `set_cached_token_counts()` for the host cached-token +counts a draft forward reads. Each of these sets the core's +`lengths_changed` and refreshes nothing derived from the lengths; a fused +kernel that writes lengths takes the buffers from +`lengths_for_kernel_update()`, which sets it too. A draft loop then calls +`metadata.refresh_draft()` on the target before the next draft forward: when +the draft's lengths changed, the draft's core re-derives its eagerly derived +length state (`resync_length_derived_state()`), every other extension runs +`refresh_from_draft_lengths()` from the draft's current lengths, and the flag +is cleared. `prepare()`, the steady decode step and the encoder-graph bind +clear it as well. Every attention forward calls `check_lengths_current()`, +which raises while it is set. The KV-cache manager is creation configuration; +`BatchInputs` carries none. A draft loop that turns the +context requests into one-token generation rows (Eagle3, MTP-Eagle, +draft-target) also clears the context cached KV through +`clear_context_cached_kv()`; MTP's correction of its draft's cached lengths +keeps the context rows. + +One-model speculative decoding runs its draft forwards on the draft metadata +`metadata.draft` of every family, owned by `DraftMetadata` on the target +(creation arguments `one_model_draft` and `draft_kv_cache_manager`). The draft +is built at creation over the draft KV cache manager (or the target's) with +the target's other extensions; its core holds no reference to the target, and +a draft over its own KV cache is created with `full_block_table_width`, so it +stages its manager's whole block table itself (DFlash and Kimi K3 read +`metadata.draft.kv_cache_block_offsets`). A spec worker runs its draft loop +inside `with metadata.draft_loop() as draft:` (`SpecWorkerBase.draft_loop`, +which also makes the draft the model's `attention_metadata` extra attribute +for MLA custom ops). At its start a draft over the target's KV cache whose +every extension declares `adopts_target_state` takes the target's prepared +state (`adopt_target_state` of each extension, by its `draft_bound` / +`draft_written` / `draft_private` lists, inside the CUDA graph the loop is +captured in); any other draft was prepared with the target from the step's +batch (with its own steady decode step after the target's) and takes only the +target's lengths, token padding and the state of the extensions that declare +`adopts_target_state_always` (speculative decoding). The draft loop changes +only the draft; the target is never saved, restored or swapped. FlashInfer's +external shared-target-KV MTP mode still drafts on a FlashInfer view +(`get_draft_metadata()`), and FlashInfer applies the overlap correction inside +the forward (`apply_spec_decode_kv_lens_offsets()`). The DSpark rerank, which +adjusts the target's ragged query and KV lengths on the device after `prepare()` +through `apply_device_ragged_layout(..., kv_lens_delta=...)` and refreshes the +sparse extension's length-derived state, is the one deliberate exception to the +target's lengths never changing after `prepare()`. DeepSeek-V4.1 decoder replay +runs the replayed decoder layers on a replay metadata of its own, built by +`CSA2Metadata` at creation and prepared from the replay plan's batch +(`prepare_decoder_replay()`). + +Two TRTLLM core methods stand in for parts of `prepare()`. A steady +speculative decode step that runs the previous step's batch one step later +calls `prepare_steady_decode_step(inputs, kv_lens)` instead, when +`supports_steady_decode_step` holds (no extension but spec-dec mode and the +draft, a draft metadata that supports it too, no cached-KV context MLA, no +mixed-precision staging): it takes the step's `BatchInputs`, adds its +`kv_lens_offsets` to the device KV lengths the engine staged (through +`lengths_for_kernel_update()`), updates the host KV lengths, re-copies block +offsets only when its manager's `batch_block_table_key` changed since +`key_prepared_block_offsets()` (or since its own last steady step), records +the step's KV lengths as `steady_decode_kv_lens`, and prepares the other +extensions; `DraftMetadata` then runs the draft's own steady step. The +context-only plan that dense layers share within one forward is kept through +`cache_context_plan()` and dropped by `prepare()`, the runtime views and a +draft refresh. A mixed-precision KV cache runs attention on a +staging copy that the core owns: `prepare()` refreshes it and takes the +corrected device KV lengths, a draft refresh keeps it in step +with the draft's lengths, a draft over the target's KV cache binds the +target's copy, and a draft over a separate mixed-precision draft manager +builds its own under the buffer names `draft_mixed_*`. An extension whose +kernels read the mixed-precision pools another way declares the field +`mixed_precision_staging_enabled` as False (DSA). + +A cross-attention sub-metadata takes its lengths from its own +`BatchInputs` (`CrossAttentionMetadata`), never from writes by its decoder. Only metadata with `supports_explicit_offsets` +(TRTLLM) honors `cu_q_seqlens` / `cu_kv_seqlens` that differ from +`cumsum(seq_lens)`; Vanilla and FlashInfer reject them. Construction-time +feature settings, such as `RecurrentConfig`, are passed to `Metadata(...)`. + +Metadata attribute assignment is strict: assigning a forwarded field writes +the extension that declares it, and assigning a name that no class in the +metadata's MRO or attached extension declares raises `AttributeError`. A family's per-step state is declared in its +core's `fields`; anything a feature or model adds belongs in an extension. + +Sparse algorithm extensions consume only `SparseMetadataParams`, not +backend-owned `SparseParams`. Algorithms that share one MTP draft-loop +selection (DSA, DeepSeek-V4, QSA) derive from `MTPIndexShareUser`, which +requires `MTPIndexShareExtension`: that extension owns the draft-loop flags, +the shared Top-K selection, the indexer page tables and the packed-row +request map, and the algorithm reaches them through `index_share`, binding the +buffers it allocates with `index_share.bind_buffers()`. + +A metadata with extensions attached is an instance of a composed subclass +of its family (one class per family and extension set) that forwards every +declared field and method, so `metadata.field` reads the attached +extension's field; `isinstance` against the family still holds, and +`core_class()` returns the family class. `ext(T)` / `find_ext(T)` remain +for presence checks and for code that keeps one extension reference. + +Every extension declares its public state in `fields` and its public +methods in `methods`. Names are unique across extension classes outside one +inheritance line (checked at class definition) and may not shadow the +metadata's own attributes or another attached extension's names (checked at +attach time). Lifecycle hooks are reserved names. **`TrtllmAttentionMetadata`** is the main metadata family. It adds paged-KV -block information, TRTLLM runtime state, chunked-prefill/speculative-decode/Helix -state, and MLA-specific state. If a source attention needs paged KV, chunked -prefill, FlashMLA, speculative decoding, or Helix-aware execution, the fit -question is mostly a `TrtllmAttentionMetadata` fit question. +block information, TRTLLM runtime state, and MLA-specific state; its features +(speculative decoding, Helix, FlashMLA, FP4 MLA, separate draft KV cache) are +extensions. If a source attention needs paged KV, chunked prefill, FlashMLA, +speculative decoding, or Helix-aware execution, the fit question is mostly a +`TrtllmAttentionMetadata` fit question. **`VanillaAttentionMetadata`** is lighter — base metadata plus simple cache-index information. Use it when the `Attention` module boundary fits but @@ -373,9 +546,9 @@ the fused TRTLLM path is too restrictive. **`FlashInferAttentionMetadata`** adds a planning-oriented contract with workspace, page-table KV metadata, and prefill/decode wrapper state. -**Sparse metadata** families extend the base backend metadata with -sparse-specific runtime state (indexer buffers, routing state, side-cache -state). +**Sparse metadata** is a `SparseAlgorithmExtension` on a core family that +holds sparse-specific runtime state (indexer buffers, routing state, +side-cache state). `SparseRuntimeParams` is the backend-to-`AttentionOp` carrier only on the `TrtllmAttention` path. Its fields are algorithm-specific, not a generic @@ -399,8 +572,8 @@ All views and that capacity are restored on exit, including exceptional exit; Preparation initializes both the indexer's per-row causal KV extents and the attention row extents on device, so prepare-to-forward callers can consume -them immediately. `on_update_kv_lens()` refreshes those same buffers in place -after device-side overlap correction. Device-selected windows must install +them immediately. With `kv_lens_offsets`, `prepare()` refreshes those same +buffers in place from the corrected device KV lengths. Device-selected windows must install their request map and KV corrections before that refresh/replay. Context attention rows start at zero; generation attention rows start at `num_contexts`, while generation tokens start at `num_ctx_tokens`. diff --git a/tensorrt_llm/_torch/attention/backends/core.py b/tensorrt_llm/_torch/attention/backends/core.py new file mode 100644 index 000000000000..4f74e427b25b --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/core.py @@ -0,0 +1,802 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The per-step batch state every attention metadata carries. + +``AttentionCoreMetadata`` is the extension a metadata attaches first: it holds +the batch layout (query and KV lengths, request and token counts, request IDs, +prompt lengths, packed-token offsets) and the per-step inputs the engine +describes the batch with. Each metadata family subclasses it with its own +state (device buffers, block offsets, planner state) and derives that state +from the batch in ``derive()``. The metadata forwards every declared field, so +``metadata.seq_lens`` reads the core's. +""" + +import contextlib +import dataclasses +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union + +import torch + +from tensorrt_llm._utils import maybe_pin_memory +from tensorrt_llm.logger import logger + +from ...memory_buffer_utils import get_memory_buffers +from .extensions import MetadataExtension + +if TYPE_CHECKING: + from ...pyexecutor.kv_cache.mixed_precision_staging import MixedPrecisionKVStaging + from ...pyexecutor.llm_request import LlmRequest + from .interface import BatchInputs + + +@dataclass +class AttentionRuntimeFeatures: + chunked_prefill: bool = False + cache_reuse: bool = False + has_speculative_draft_tokens: bool = False + # This is the chunk size for MLA chunked prefill, which splits KV cache into chunks. + chunk_size: int = 0 + # The real chunk size for MLA chunked prefill is this value * chunk_size. + chunked_prefill_buffer_batch_size: int = 4 + + +class AttentionCoreMetadata(MetadataExtension): + """Batch layout and per-step inputs of an attention metadata.""" + + is_core = True + fields = frozenset( + { + "max_num_requests", + "max_num_tokens", + "max_num_sequences", + "mapping", + "kv_cache_manager", + "kv_layout", + "runtime_features", + "is_cuda_graph", + "enable_context_mla_with_cached_kv", + "cache_indirection", + "cuda_graph_buffers", + "full_block_table_width", + "mixed_precision_staging_name", + "mixed_precision_staging", + "seq_lens", + "seq_lens_cuda", + "seq_lens_kv", + "seq_lens_kv_cuda", + # The length buffers and counts behind the properties above, which + # callers outside the core still read and write directly. + "_seq_lens", + "_seq_lens_cuda", + "_seq_lens_kv", + "_seq_lens_kv_cuda", + "_num_contexts", + "_num_generations", + "_num_ctx_tokens", + "_num_tokens", + "num_contexts", + "num_generations", + "num_ctx_tokens", + "num_tokens", + "num_seqs", + "context_lens", + "is_cross", + "position_ids", + "kv_cache_params", + "request_ids", + "prompt_lens", + "cu_q_seqlens", + "cu_kv_seqlens", + "all_rank_num_tokens", + "padded_num_tokens", + "num_chunked_ctx_requests", + "beam_width", + "multi_item_part_lens", + "context_requests", + "prefill_cuda_graphs", + "kv_lens_offsets", + "position_id_offsets", + "lengths_changed", + "steady_decode_kv_lens", + } + ) + methods = frozenset( + { + "clear_lengths_changed", + "lengths_for_kernel_update", + "resync_length_derived_state", + "on_update", + "bind_lengths", + "mark_cuda_graph", + "set_all_rank_num_tokens", + "set_padded_num_tokens", + "set_context_requests", + "override_cached_token_lens", + "set_cached_token_counts", + "set_seq_lens", + "adjust_seq_lens", + "sync_seq_lens_to_device", + "copy_lengths_from", + "adjust_kv_lens", + "set_kv_lens", + "as_generation_rows", + "prepare_mixed_precision_kv", + "get_mixed_precision_kv_view", + "mixed_precision_kv_staging", + } + ) + init_fields = { + # Capacities: requests, tokens and sequences (requests times beams) in + # one batch. The family fills in max_num_sequences when unset. + "max_num_requests": 0, + "max_num_tokens": 0, + "max_num_sequences": None, + "mapping": None, + # The KV cache manager; None for attention without a KV cache. + "kv_cache_manager": None, + # Paged KV-cache block layout: + # NHD: [max_num_pages, 2, page_size, num_kv_heads, head_dim] + # HND: [max_num_pages, 2, num_kv_heads, page_size, head_dim] + "kv_layout": "HND", + # Features the runtime may use; kernels may behave differently with them. + "runtime_features": AttentionRuntimeFeatures(), + # Whether this metadata is a CUDA graph's: buffers are bound once and + # updated in place. + "is_cuda_graph": False, + "enable_context_mla_with_cached_kv": False, + # The beam-search cache indirection buffer TRTLLM attention reads. + "cache_indirection": None, + # The buffer pool persistent buffers come from; None allocates directly. + "cuda_graph_buffers": None, + # Whether every step stages the whole block table, as for a metadata + # whose device KV lengths run past the host snapshot between prepares + # (a one-model draft over its own KV cache). + "full_block_table_width": False, + # Buffer-name prefix of the mixed-precision staging copy; metadata over + # different KV cache managers in one buffer pool need different ones. + "mixed_precision_staging_name": "mixed", + "seq_lens": None, + "num_contexts": 0, + # Position of each token in each sequence; None when positional + # embedding is applied outside the backend. + "position_ids": None, + "kv_cache_params": None, + "seq_lens_kv": None, + # Request ID and prompt length of each sequence. A context sequence's + # prompt length is its token count; a generation sequence's is the + # token count of its context phase. + "request_ids": None, + "prompt_lens": None, + # Explicit query/KV sequence boundaries [num_seqs + 1] for kernels on + # packed varlen inputs; sequence or segment boundaries, not + # necessarily request boundaries. + "cu_q_seqlens": None, + "cu_kv_seqlens": None, + # Token count of each attention-DP rank, and this rank's padded count. + "all_rank_num_tokens": None, + "padded_num_tokens": None, + "beam_width": 1, + # Per-request multi-item scoring layout; see ``TokensPrompt``. + "multi_item_part_lens": None, + } + + # Storage behind the length and count properties, read by their setters + # while the base constructor sets the init fields. + _seq_lens: Optional[torch.Tensor] = None + _seq_lens_kv: Optional[torch.Tensor] = None + _seq_lens_cuda: Optional[torch.Tensor] = None + _seq_lens_kv_cuda: Optional[torch.Tensor] = None + _num_contexts = 0 + _num_generations = 0 + _num_ctx_tokens = 0 + _num_tokens = 0 + + #: Whether the family's kernels address the paged KV cache through + #: attention-op block offsets, which the draft KV cache then also needs. + pages_by_block_offsets: ClassVar[bool] = False + #: Constructor arguments every metadata must be given. + required_init_fields: ClassVar[Tuple[str, ...]] = ("max_num_requests", "max_num_tokens") + # Copies share the pinned staging rings and the mixed-precision staging + # cache (built from the shared buffer pool on the first prepare). + shared = frozenset({"_host_staging", "mixed_precision_staging"}) + # A draft binds the target's batch description, counts and mixed-precision + # staging copy, writes its query lengths in place, and keeps its KV cache + # managers, buffer pool, pinned staging rings and saved length tensors. + draft_bound = frozenset( + { + "max_num_requests", + "max_num_tokens", + "max_num_sequences", + "mapping", + "kv_layout", + "runtime_features", + "enable_context_mla_with_cached_kv", + "cache_indirection", + "_seq_lens_kv", + "_seq_lens_kv_cuda", + "_num_contexts", + "_num_generations", + "_num_ctx_tokens", + "_num_tokens", + "position_ids", + "kv_cache_params", + "request_ids", + "prompt_lens", + "cu_q_seqlens", + "cu_kv_seqlens", + "all_rank_num_tokens", + "padded_num_tokens", + "num_chunked_ctx_requests", + "beam_width", + "multi_item_part_lens", + "context_requests", + "prefill_cuda_graphs", + "kv_lens_offsets", + "position_id_offsets", + # Over the target's KV cache the draft runs on the target's + # staging copy; a separate draft manager's is built by the draft. + "mixed_precision_staging", + } + ) + draft_written = frozenset({"_seq_lens", "_seq_lens_cuda"}) + draft_private = frozenset( + { + "kv_cache_manager", + "cuda_graph_buffers", + "is_cuda_graph", + "_host_staging", + "lengths_changed", + "steady_decode_kv_lens", + "full_block_table_width", + "mixed_precision_staging_name", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Pinned staging rings for stage_to_device, shared with metadata copies. + self._host_staging: Dict[Any, Any] = self.shared_or("_host_staging", dict) + # Mixed-precision KV: the staging copy attention reads, created by the + # first prepare() on a manager with mixed-precision layers. + self.mixed_precision_staging: Optional["MixedPrecisionKVStaging"] = self.shared_or( + "mixed_precision_staging", lambda: None + ) + # Host per-sequence query / KV lengths and their device copies (the KV + # lengths default to the query lengths unless set, as for cross + # attention), and the counts derived on the host whenever the lengths + # or the context count change, so compiled code never computes them + # from tensors. The length setters above may have set them already. + for name in ( + "_seq_lens", + "_seq_lens_kv", + "_seq_lens_cuda", + "_seq_lens_kv_cuda", + "_num_contexts", + "_num_generations", + "_num_ctx_tokens", + "_num_tokens", + ): + self.__dict__.setdefault(name, getattr(type(self), name)) + # Trailing context requests that are speculative-decoding extend + # requests; their device KV lengths are corrected with the generation + # rows. + self.num_chunked_ctx_requests = 0 + # This step's scheduled context requests, or None when not given, and + # whether context forwards may run in CUDA graphs. + self.context_requests: Optional[List["LlmRequest"]] = None + self.prefill_cuda_graphs = False + # This step's device corrections to the KV lengths + # (``BatchInputs.kv_lens_offsets``), or None. + self.kv_lens_offsets: Optional[torch.Tensor] = None + # This step's device corrections to the generation tokens' positions + # (``BatchInputs.position_id_offsets``), or None. + self.position_id_offsets: Optional[torch.Tensor] = None + # Whether a length primitive changed the lengths in place since the + # last prepare or refresh: state derived from them is stale, and the + # attention forward refuses to run. + self.lengths_changed = False + # The scheduled KV lengths of a steady decode step + # (``prepare_steady_decode_step``); None after a full prepare. + self.steady_decode_kv_lens: Optional[List[int]] = None + + def derive(self) -> None: + """Derive the family's per-step state from the batch; runs in ``prepare``.""" + + def prepare_encoder_only(self) -> None: + """Encoder-only (no-KV-cache) setup; defaults to the full ``prepare()``.""" + self.metadata.prepare() + + def on_update(self) -> None: + """Recompute the token and request counts from the host lengths.""" + if ( + self._seq_lens is not None + and self._seq_lens.shape[0] >= self._num_contexts + and self._num_contexts >= 0 + ): + self._num_ctx_tokens = self._seq_lens[: self._num_contexts].sum().item() + self._num_generations = self._seq_lens.shape[0] - self._num_contexts + if self._seq_lens_kv is not None: + self._num_tokens = self._seq_lens_kv.sum().item() + elif self._seq_lens is not None: + self._num_tokens = self._seq_lens.sum().item() + + @property + def seq_lens(self) -> Optional[torch.Tensor]: + return self._seq_lens + + @seq_lens.setter + def seq_lens(self, value: Optional[torch.Tensor]) -> None: + self._seq_lens = value + self.on_update() + + # The model executor sets seq_lens to None initially. + if self._seq_lens is not None: + self._seq_lens = maybe_pin_memory(self._seq_lens) + + if self.is_cuda_graph and self._seq_lens_cuda is not None: + # Very important: do not reallocate if we are using CUDA graphs. + # This copy is safe because the batch size is guaranteed to not + # change in the CUDA graph case. The seqlens can change if we + # are doing spec decode. + self._seq_lens_cuda.copy_(self._seq_lens, non_blocking=True) + else: + self._seq_lens_cuda = self._seq_lens.cuda(non_blocking=True) + + @property + def num_contexts(self) -> int: + return self._num_contexts + + @num_contexts.setter + def num_contexts(self, value: int) -> None: + self._num_contexts = value + self.on_update() + + @property + def num_generations(self) -> int: + return self._num_generations + + @num_generations.setter + def num_generations(self, value: int) -> None: + self._num_generations = value + self.on_update() + + @property + def seq_lens_cuda(self) -> Optional[torch.Tensor]: + return self._seq_lens_cuda + + @property + def seq_lens_kv(self) -> Optional[torch.Tensor]: + return self._seq_lens_kv if self._seq_lens_kv is not None else self._seq_lens + + @seq_lens_kv.setter + def seq_lens_kv(self, value: Optional[torch.Tensor]) -> None: + self._seq_lens_kv = value + self.on_update() + # The model executor sets seqlens to None initially. + if self._seq_lens_kv is not None: + self._seq_lens_kv = maybe_pin_memory(self._seq_lens_kv) + if self.is_cuda_graph and self._seq_lens_kv_cuda is not None: + self._seq_lens_kv_cuda.copy_(self._seq_lens_kv, non_blocking=True) + else: + self._seq_lens_kv_cuda = self._seq_lens_kv.cuda(non_blocking=True) + + @property + def seq_lens_kv_cuda(self) -> Optional[torch.Tensor]: + if self._seq_lens_kv_cuda is not None: + return self._seq_lens_kv_cuda + return self._seq_lens_cuda + + @property + def context_lens(self) -> torch.Tensor: + """The query length of each context sequence, [num_contexts].""" + return self.seq_lens[: self.num_contexts] + + @property + def num_seqs(self) -> int: + """The number of sequences in the batch.""" + return self.seq_lens.shape[0] + + @property + def is_cross(self) -> bool: + """Whether this metadata is for cross attention.""" + return self.seq_lens is not self.seq_lens_kv + + @property + def num_ctx_tokens(self) -> int: + return self._num_ctx_tokens + + @property + def num_tokens(self) -> int: + return self._num_tokens + + # Length changes. The lengths and counts belong to the core; everything + # else changes them through these methods, which keep the device buffers + # in place and set ``lengths_changed``. Extensions correct them only from + # their own ``prepare`` (``prepare`` clears the flag after every extension + # prepared); after prepare only draft loops change them, on their draft + # metadata, and then refresh the draft before its next forward. + + def clear_lengths_changed(self) -> None: + """Mark the state derived from the lengths as current.""" + self.lengths_changed = False + + def resync_length_derived_state(self) -> None: + """Re-derive the family state that follows the lengths eagerly. + + For a draft refresh after in-place length changes; state derived + lazily is invalidated by the length primitives themselves. + """ + + def lengths_for_kernel_update(self) -> Dict[str, torch.Tensor]: + """The device length buffers, for a fused kernel that updates them. + + The only way a kernel may write the lengths: the buffers are marked + changed, so the next forward sees them refreshed. Keys are + ``seq_lens_cuda`` and, for a family with device KV lengths, + ``kv_lens_cuda``. + """ + self.lengths_changed = True + buffers = {"seq_lens_cuda": self._seq_lens_cuda} + kv_lens = getattr(self, "kv_lens_cuda", None) + if kv_lens is not None: + buffers["kv_lens_cuda"] = kv_lens + return buffers + + def apply_batch_inputs(self, inputs: "BatchInputs") -> None: + """Take one step's batch description; ``prepare`` derives from it.""" + if inputs.seq_lens_cuda is not None: + self.bind_lengths(inputs.seq_lens, inputs.seq_lens_cuda) + elif inputs.seq_lens is not None: + self.seq_lens = inputs.seq_lens + if inputs.seq_lens_kv is not None: + self.seq_lens_kv = inputs.seq_lens_kv + self.beam_width = inputs.beam_width + self.request_ids = inputs.request_ids + self.prompt_lens = inputs.prompt_lens + self.num_contexts = inputs.num_contexts + self.num_chunked_ctx_requests = inputs.num_chunked_ctx_requests + self.kv_cache_params = inputs.kv_cache_params + self.cu_q_seqlens = inputs.q_offsets + self.cu_kv_seqlens = inputs.kv_offsets + self.kv_lens_offsets = inputs.kv_lens_offsets + self.position_id_offsets = inputs.position_id_offsets + self.multi_item_part_lens = inputs.multi_item_part_lens + + def bind_lengths( + self, + host: torch.Tensor, + device: Optional[torch.Tensor] = None, + *, + num_contexts: Optional[int] = None, + num_ctx_tokens: Optional[int] = None, + num_generations: Optional[int] = None, + num_tokens: Optional[int] = None, + ) -> None: + """Bind the query lengths as given, without copying. + + ``device`` is bound as the device copy (shared with its owner, such as + a parent metadata or a fixed-geometry tile); None stages ``host`` into + the bound device buffer of a CUDA graph, or a new one. Counts given are + taken as they are, + so a caller that already knows them skips the host reductions; the + others are recomputed from ``host``. + """ + self._seq_lens = host + if device is not None: + self._seq_lens_cuda = device + elif self.is_cuda_graph and self._seq_lens_cuda is not None: + # Keep the graph's buffer; the batch size is fixed in a graph. + self._seq_lens_cuda.copy_(host, non_blocking=True) + else: + self._seq_lens_cuda = host.cuda(non_blocking=True) + if num_contexts is not None: + self._num_contexts = num_contexts + counts = (num_ctx_tokens, num_generations, num_tokens) + if all(count is not None for count in counts): + self._num_ctx_tokens, self._num_generations, self._num_tokens = counts + else: + self.on_update() + + def set_context_requests( + self, requests: Optional[List["LlmRequest"]], prefill_cuda_graphs: bool + ) -> None: + """Take this step's scheduled context requests (None when not given).""" + self.context_requests = requests + self.prefill_cuda_graphs = prefill_cuda_graphs + + def override_cached_token_lens(self, num_cached_tokens_per_seq: List[int]) -> None: + """Present other cached-token counts than the step's KV-cache parameters. + + For an extension that attends to fewer cached tokens than the batch + holds (RocketKV's prompt budget); called from its ``prepare`` after + the core derived. The step's ``kv_cache_params`` object is replaced, + never changed in place; a family that derives KV lengths from the + counts re-derives them. + """ + self.kv_cache_params = dataclasses.replace( + self.kv_cache_params, num_cached_tokens_per_seq=list(num_cached_tokens_per_seq) + ) + + def set_cached_token_counts(self, num_cached_tokens_per_seq: List[int]) -> None: + """Present other host cached-token counts to this metadata's forwards. + + For a draft loop whose draft forwards keep fewer cached tokens than + the step's KV-cache parameters count. The ``kv_cache_params`` object, + which a draft metadata shares with its target, is replaced, never + changed in place; nothing is re-derived, so the device KV lengths + stay as the caller set them. It counts as a length change. + """ + self.kv_cache_params = dataclasses.replace( + self.kv_cache_params, num_cached_tokens_per_seq=list(num_cached_tokens_per_seq) + ) + self.lengths_changed = True + + def set_all_rank_num_tokens(self, counts: Optional[List[int]]) -> None: + """Set the token count of each attention-DP rank.""" + self.all_rank_num_tokens = counts + + def set_padded_num_tokens(self, padded_num_tokens: Optional[int]) -> None: + """Set the token count the model runs on when the batch is padded (None: unpadded). + + The padding is chosen after ``prepare``, from the prepared token + counts of every attention-DP rank. + """ + self.padded_num_tokens = padded_num_tokens + + def mark_cuda_graph(self, capturing: bool) -> None: + """Mark whether this metadata's buffers are bound for graph capture.""" + self.is_cuda_graph = capturing + + def set_seq_lens( + self, + rows: slice, + *, + host: Union[int, torch.Tensor, None] = None, + device: Union[int, torch.Tensor, None] = None, + update_counts: bool = False, + ) -> None: + """Overwrite the query lengths of ``rows`` in place, host and/or device. + + The buffers keep their addresses, so captured graphs stay valid. With + ``update_counts`` the token and request counts are recomputed from the + host lengths; otherwise the step's counts stay. + """ + if host is not None: + self._seq_lens[rows] = host + if device is not None: + self._seq_lens_cuda[rows] = device + if update_counts: + self.on_update() + self.lengths_changed = True + + def adjust_seq_lens(self, rows: slice, delta: int) -> None: + """Add ``delta`` to the host and device query lengths of ``rows``. + + The token and request counts are recomputed from the new lengths. + """ + self._seq_lens[rows] += delta + self._seq_lens_cuda[rows] += delta + self.on_update() + self.lengths_changed = True + + def copy_lengths_from(self, source: "AttentionCoreMetadata") -> None: + """Start this metadata's lengths and counts from ``source``'s, in place. + + A draft loop starts a draft metadata from its target's lengths this + way, inside the captured graph, so repeated warmup and capture runs + begin from the same lengths. The token padding the engine set on the + target after its ``prepare`` is taken as well. + """ + if ( + self._seq_lens is not None + and self._seq_lens is not source._seq_lens + and self._seq_lens.shape == source._seq_lens.shape + ): + self._seq_lens.copy_(source._seq_lens) + else: + self._seq_lens = source._seq_lens.clone() + self._num_contexts = source._num_contexts + self._num_generations = source._num_generations + self._num_ctx_tokens = source._num_ctx_tokens + self._num_tokens = source._num_tokens + num_seqs = self.num_seqs + self._seq_lens_cuda[:num_seqs].copy_(source.seq_lens_cuda[:num_seqs]) + self.padded_num_tokens = source.padded_num_tokens + self.all_rank_num_tokens = source.all_rank_num_tokens + self.lengths_changed = True + + def sync_seq_lens_to_device(self) -> None: + """Copy the host query lengths into the bound device buffer in place. + + The device copy takes the host lengths the state was derived from (an + encoder CUDA graph's prepare sets them on the host), so this is no + length change. + """ + self._seq_lens_cuda.copy_(self._seq_lens, non_blocking=True) + + def adjust_kv_lens( + self, + rows: slice, + delta: Union[int, torch.Tensor], + *, + min_value: Optional[int] = None, + ) -> None: + """Add ``delta`` to the device KV lengths of ``rows`` in place. + + ``kv_lens_cuda`` ([max_num_sequences], int32, without the extra KV + tokens) of a family that keeps device KV lengths. With ``min_value`` + the new lengths are clamped from below. Nothing derived from the + lengths is refreshed here; the draft is refreshed before its next + forward. + """ + kv_lens = self.kv_lens_cuda[rows] + kv_lens += delta + if min_value is not None: + kv_lens.clamp_(min=min_value) + self.lengths_changed = True + + def set_kv_lens(self, rows: slice, values: torch.Tensor) -> None: + """Overwrite the device KV lengths of ``rows`` in place; see ``adjust_kv_lens``.""" + self.kv_lens_cuda[rows].copy_(values) + self.lengths_changed = True + + def as_generation_rows(self) -> None: + """Continue every context row of a draft metadata as a generation row. + + For the first step of a one-model draft loop: the context requests + drafted their first token from the prompt and, from the next draft + forward on, run like the generation requests. The context count becomes + zero and the token and request counts are recomputed from the host + lengths, which the draft loop sets first (``set_seq_lens``). A family + core also drops the state it keeps only for context rows; the next + draft refresh sees the new rows. + """ + self._num_contexts = 0 + self.on_update() + self.lengths_changed = True + + def cuda_graph_copy_changes(self, max_batch_size: int, max_draft_tokens: int) -> Dict[str, Any]: + """Creation arguments a CUDA graph copy changes besides the graph flag and pool.""" + return {} + + def creation_value(self, name: str) -> Any: + # The stored lengths, not the getters' fallbacks: KV lengths default + # to the query lengths only while unset. + if name in ("seq_lens", "seq_lens_kv"): + return self.__dict__[f"_{name}"] + return super().creation_value(name) + + def reset_cuda_graph_lengths( + self, + max_batch_size: int, + *, + max_draft_tokens: int, + sub_cross_metadata: bool, + encode_only: bool, + ) -> None: + """Bind graph-sized length buffers on a CUDA graph copy. + + The device query lengths are reallocated at the graph's batch size; + a cross-attention sub-metadata keeps its parent's. Encoder graphs vary + the query lengths per replay; decoder graphs are generation-only, so + their context count is zero (encoder graphs set it per replay). + """ + if not sub_cross_metadata: + self._seq_lens_cuda = None + if encode_only: + self.seq_lens = torch.ones((max_batch_size,), dtype=torch.int) + else: + self.seq_lens = torch.ones((max_batch_size,), dtype=torch.int) * ( + 1 + max_draft_tokens + ) + if self.is_cross: + self.seq_lens_kv = torch.zeros((max_batch_size,), dtype=torch.int) + if not encode_only: + self.num_contexts = 0 + + def _has_mixed_precision_staging(self) -> bool: + """Whether attention runs on a staging copy of the mixed-precision layers. + + An extension whose kernels read the mixed-precision pools another way + declares the field ``mixed_precision_staging_enabled`` as False (DSA + reads its gather's FP8 scratch). + """ + if not self.metadata.find("mixed_precision_staging_enabled", True): + return False + # isinstance guards against mocks and against V1 managers, which have + # no mixed-precision layers. + high_layers = getattr(self.kv_cache_manager, "mixed_precision_high_layers", None) + return isinstance(high_layers, dict) and bool(high_layers) + + def prepare_mixed_precision_kv(self, layout: str, tokens_per_block: int) -> None: + """Create the staging copy on first use, then refresh it per batch. + + The staging copy is of this metadata's KV cache manager; a one-model + draft over a separate draft manager (``md.draft``) builds its own, + under buffer names of its own. + """ + from ...pyexecutor.kv_cache.mixed_precision_staging import MixedPrecisionKVStaging + + name = self._mixed_precision_staging_name() + view = self.mixed_precision_staging + if view is None or view.manager is not self.kv_cache_manager: + # Built once per metadata object (eager or CUDA graph copy); the + # reserved buffer-pool block is shared, so the staging copy is + # allocated and zeroed once per rank and the KV budget profiling + # run already accounts for it. + buffers = ( + self.cuda_graph_buffers + if self.cuda_graph_buffers is not None + else get_memory_buffers() + ) + view = MixedPrecisionKVStaging( + self.kv_cache_manager, + buffers, + rows=self.max_num_sequences or self.max_num_requests, + layout=layout, + pool_layout=self.kv_layout, + tokens_per_block=tokens_per_block, + name=name, + ) + logger.info_once( + f"Mixed-precision KV staging ({name}) on this rank: " + f"{view.staging_bytes / (1 << 20):.1f} MiB = {view.rows} " + f"requests x {view.max_blocks_per_seq} pages x " + f"{view.max_page_bytes} B ({layout}, {tokens_per_block} " + "tokens per page)", + key=f"{name}_precision_kv_staging", + ) + self.mixed_precision_staging = view + view.update( + self.request_ids, + self.kv_cache_params.num_cached_tokens_per_seq, + self.seq_lens.tolist(), + capture_graph=self.is_cuda_graph, + ) + self._on_mixed_precision_kv_prepared(view) + + def _mixed_precision_staging_name(self) -> str: + """Buffer-name prefix of this metadata's staging copy (creation setting).""" + return self.mixed_precision_staging_name + + def _on_mixed_precision_kv_prepared(self, view: "MixedPrecisionKVStaging") -> None: + """Backend hook: refresh backend-specific views of the staging cache.""" + + def get_mixed_precision_kv_view(self, layer_idx: int) -> Optional["MixedPrecisionKVStaging"]: + """Return the staging cache of a mixed-precision layer, else None.""" + staging = self.mixed_precision_staging + if ( + staging is None + or layer_idx not in staging.pools + or staging.manager is not self.kv_cache_manager + ): + # No staging for a layer without a mixed-precision high pool, or of a manager not in use. + return None + return staging + + @contextlib.contextmanager + def mixed_precision_kv_staging(self, layer_idx: int): + """Run attention on the staging copy, then write the new tokens back.""" + view = self.get_mixed_precision_kv_view(layer_idx) + if view is None: + yield + return + view.convert(layer_idx) + saved = self._swap_mixed_precision_kv(view) + try: + yield + finally: + self._restore_mixed_precision_kv(saved) + view.convert(layer_idx, writeback=True) + + def _swap_mixed_precision_kv(self, view: "MixedPrecisionKVStaging") -> tuple: + """Point this metadata at the staging cache; return what to restore.""" + saved = (self.kv_cache_manager, self.kv_layout) + self.kv_cache_manager = view + self.kv_layout = view.layout + return saved + + def _restore_mixed_precision_kv(self, saved: tuple) -> None: + """Undo _swap_mixed_precision_kv.""" + self.kv_cache_manager, self.kv_layout = saved diff --git a/tensorrt_llm/_torch/attention/backends/extensions.py b/tensorrt_llm/_torch/attention/backends/extensions.py new file mode 100644 index 000000000000..cdbb88f9a957 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/extensions.py @@ -0,0 +1,695 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Feature state attached to attention metadata. + +An ``AttentionMetadata`` holds the batch layout every attention backend needs: +request counts, sequence and KV lengths, the paged KV cache and the workspace. +Everything a feature, a sparse algorithm or a model adds on top lives in a +``MetadataExtension`` attached to it. The metadata drives each extension +through the hooks below at fixed points of its lifecycle, and callers reach an +extension's state with ``metadata.ext(T)`` or ``metadata.find_ext(T)``. + +Every extension declares the public state it exposes in ``fields`` and the +public methods in ``methods``. Names are unique: no two extension classes +outside one inheritance line declare the same name (checked when a class is +defined), and an extension's names may not shadow its metadata's own +attributes or another attached extension's (checked when it is attached). +Lifecycle hooks are reserved and never declared. + +Lifecycle, driven by the metadata: + +- The constructor (``__init__(metadata, *, source=None, **config)``) calls + ``super().__init__`` first, which sets every ``init_fields`` entry from the + routed creation arguments or its declared default; it then sets the + extension's other state and allocates its persistent buffers from the + metadata's pool, and may read the extensions constructed before it. Every + CUDA graph, cross-attention or draft metadata is constructed anew from the + same configuration; an attribute the extension declares in ``shared`` comes + from ``shared_or(name, allocate)``, the source's object on a copy. +- ``prepare()`` runs once per step after the core derived, in dependency + order, and every forward of the metadata follows its own ``prepare``. An + extension derives its own state there and corrects the core's only through + core methods. When the core applied ``kv_lens_offsets`` (corrections known + only on device) or an extension prepared earlier corrected them on device + (Helix's rank-local lengths of speculative verify groups), the device KV + lengths differ from the host values, and an extension derives its + length-dependent device state from the device lengths. + +Values the core or the engine reads from whichever extension provides them +(``kv_cache_pools``, ``full_block_table_width``, ``adp_input_token_count``, +``adp_token_counts``, +``token_major_gen_view``, ``mla_scheduler_buffers``) +are ordinary declared fields and methods, read with ``metadata.find(name, +default)``. + +Opt-in members, called only by ``DraftMetadata`` on the extensions of a +one-model draft metadata (``md.draft``): + +- ``adopt_target_state(source)`` takes the target's prepared state at the + start of a draft loop (by the ``draft_*`` lists; an override calls + ``super()`` first and fixes up what the lists cannot express). +- ``refresh_from_draft_lengths()`` re-derives the state that depends on the + draft's lengths after the draft loop changed them in place, and resets the + state the previous draft forward left. ``DraftMetadata.refresh_draft()`` + calls it before a draft forward, only when the draft's lengths changed. + +Length-derived state an extension computes lazily is invalidated by the +extension itself, in its ``prepare`` and ``refresh_from_draft_lengths``. +""" + +from __future__ import annotations + +import copy +import operator +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + FrozenSet, + Iterable, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, +) + +import torch + +if TYPE_CHECKING: + from .interface import AttentionMetadata + +E = TypeVar("E", bound="MetadataExtension") + +# Containers a draft copies when it adopts its target's state, so that +# clearing or filling its own never changes the target's. +_COPIED_CONTAINERS = (dict, set) +# Dtypes torch._foreach_copy_ has no kernel for; copied one tensor at a time. +_NO_FOREACH_COPY_DTYPES = frozenset({torch.uint16, torch.uint32, torch.uint64}) +# Attributes the adoption itself keeps on a draft's extension. +_ADOPTION_BOOKKEEPING = frozenset({"metadata", "_draft_copy_plan", "_draft_adopted_names"}) +# The attribute a copy's extension keeps its source in while it is constructed. +_COPY_SOURCE = "_copy_source" + + +# Every name an extension class declares in ``fields`` / ``methods``, mapped to +# the class that declared it first. +_DECLARED_NAMES: Dict[str, type] = {} +# Every metadata constructor argument, mapped to the extension class that takes it. +_INIT_FIELD_OWNERS: Dict[str, type] = {} + + +class MetadataExtension: + """State that one feature, sparse algorithm or model attaches to a metadata. + + Subclasses keep their state as attributes and override only the hooks they + need; every hook defaults to doing nothing. ``self.metadata`` is the + metadata the extension is attached to; read the core through it and never + assign the core's private fields. + """ + + #: Extensions that must be attached, and prepared, before this one. They + #: are attached automatically when this one is. + requires: ClassVar[Tuple[Type["MetadataExtension"], ...]] = () + #: Tensor attributes produced and consumed inside a captured graph; the + #: strict CUDA-graph buffer check skips them. + graph_temporary_attrs: ClassVar[FrozenSet[str]] = frozenset() + #: Public state this class adds; subclasses inherit their bases' fields. + fields: ClassVar[FrozenSet[str]] = frozenset() + #: Public methods this class adds, other than lifecycle hooks. + methods: ClassVar[FrozenSet[str]] = frozenset() + #: Whether this is a metadata family's core state; exactly one core is + #: attached to each metadata, before every other extension. + is_core: ClassVar[bool] = False + #: Metadata constructor arguments this class takes, each mapped to its + #: default; each is one of its declared fields. A mutable default is + #: copied per extension. + init_fields: ClassVar[Dict[str, Any]] = {} + #: Whether this extension is constructed after every other one, because + #: its constructor needs the complete metadata. + constructed_last: ClassVar[bool] = False + #: Attributes a copy of the metadata (a CUDA graph or cross-attention + #: metadata) takes by reference from its source's extension instead of + #: allocating them: kernel workspaces, buffers written on the source each + #: step and caches every copy must see. + shared: ClassVar[FrozenSet[str]] = frozenset() + #: Whether a draft metadata takes this extension's prepared state from its + #: target's (``adopt_target_state``) instead of deriving it again. A draft + #: reuses the target's state only when every extension attached to it does. + adopts_target_state: ClassVar[bool] = False + #: Whether a draft takes this extension's state from its target's at every + #: draft-loop start, also a draft that derives the rest itself: state that + #: depends on neither the KV cache nor other extensions. + adopts_target_state_always: ClassVar[bool] = False + #: Attributes a draft adopting the target's state binds to the target's: + #: the prepared state it reads but never writes. + draft_bound: ClassVar[FrozenSet[str]] = frozenset() + #: Tensors a draft loop writes in place. A draft adopting the target's + #: state keeps its own and copies the target's contents into them. + draft_written: ClassVar[FrozenSet[str]] = frozenset() + #: Attributes a draft adopting the target's state keeps as they are: its + #: configuration, KV cache managers, workspaces and lazily built buffers. + #: Anything in none of the three lists is also kept; the contract tests + #: require every attribute of an adopting extension to be listed. + draft_private: ClassVar[FrozenSet[str]] = frozenset() + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + own = cls._own_declared_names() + reserved = own & reserved_extension_names() + if reserved: + raise TypeError(f"{cls.__name__} declares reserved names {sorted(reserved)}") + overlap = cls.__dict__.get("fields", frozenset()) & cls.__dict__.get("methods", frozenset()) + if overlap: + raise TypeError(f"{cls.__name__} declares {sorted(overlap)} as both field and method") + for name in cls.__dict__.get("init_fields", ()): + if name not in cls.declared_fields(): + raise TypeError(f"{cls.__name__} takes {name!r} but does not declare it a field") + _INIT_FIELD_OWNERS.setdefault(name, cls) + if cls.is_core: + # A core is its family's own state, checked against the extensions + # attached with it rather than against every extension class. + return + for name in own: + owner = _DECLARED_NAMES.get(name) + if owner is None or issubclass(cls, owner) or issubclass(owner, cls): + continue + # A reloaded module redefines the same class. + if (owner.__module__, owner.__qualname__) != (cls.__module__, cls.__qualname__): + raise TypeError( + f"{cls.__name__} declares {name!r}, which {owner.__name__} already declares" + ) + for name in own: + owner = _DECLARED_NAMES.get(name) + if owner is None or not issubclass(cls, owner): + _DECLARED_NAMES[name] = cls + + @classmethod + def _own_declared_names(cls) -> FrozenSet[str]: + return cls.__dict__.get("fields", frozenset()) | cls.__dict__.get("methods", frozenset()) + + @classmethod + def declared_fields(cls) -> FrozenSet[str]: + """Every field this class and its bases declare.""" + return frozenset().union( + *(vars(klass).get("fields", ()) for klass in cls.__mro__ if klass is not object) + ) + + @classmethod + def declared_methods(cls) -> FrozenSet[str]: + """Every public method this class and its bases declare.""" + return frozenset().union( + *(vars(klass).get("methods", ()) for klass in cls.__mro__ if klass is not object) + ) + + @classmethod + def declared_names(cls) -> FrozenSet[str]: + return cls.declared_fields() | cls.declared_methods() + + @classmethod + def declared_init_fields(cls) -> Tuple[str, ...]: + """Every constructor argument this class and its bases take, bases first.""" + return tuple(cls.init_field_defaults()) + + @classmethod + def init_field_defaults(cls) -> Dict[str, Any]: + """Every constructor argument this class and its bases take, with its + default, bases first; a subclass's default replaces its base's.""" + defaults: Dict[str, Any] = {} + for klass in reversed(cls.__mro__): + defaults.update(vars(klass).get("init_fields", {})) + return defaults + + def __init__( + self, + metadata: "AttentionMetadata", + *, + source: Optional["MetadataExtension"] = None, + **config: Any, + ) -> None: + """Construct this extension of ``metadata``. + + Sets every ``init_fields`` entry of the class and its bases from + ``config`` (the metadata constructor arguments routed to this + extension) or from its declared default. A subclass calls this first, + then sets its other state and allocates its buffers from the + metadata's pool (``metadata.cuda_graph_buffers``, graph-stable on a + CUDA graph metadata), so it allocates after its base. ``source`` is + the same extension of the metadata this one's is a copy of; see + ``shared_or``. + """ + self.metadata = metadata + defaults = type(self).init_field_defaults() + unknown = set(config) - set(defaults) + if unknown: + raise TypeError(f"{type(self).__name__} takes no {sorted(unknown)}") + if source is not None: + self.__dict__[_COPY_SOURCE] = source + for name, default in defaults.items(): + setattr(self, name, config[name] if name in config else copy.copy(default)) + + def shared_or(self, name: str, allocate: Callable[[], Any]) -> Any: + """The source's ``name`` on a copy of a metadata, else ``allocate()``. + + For the attributes the class declares in ``shared``, set from its + constructor. + """ + if name not in type(self).declared_shared(): + raise AttributeError(f"{type(self).__name__} does not declare {name!r} shared") + source = self.__dict__.get(_COPY_SOURCE) + if source is not None and name in vars(source): + return vars(source)[name] + return allocate() + + @classmethod + def declared_shared(cls) -> FrozenSet[str]: + """Every attribute this class and its bases share with copies.""" + return frozenset().union( + *(vars(klass).get("shared", ()) for klass in cls.__mro__ if klass is not object) + ) + + @classmethod + def _declared_union(cls, name: str) -> FrozenSet[str]: + return frozenset().union( + *(vars(klass).get(name, ()) for klass in cls.__mro__ if klass is not object) + ) + + @classmethod + def _adoption_names(cls) -> Tuple[FrozenSet[str], Tuple[str, ...]]: + """The attributes a draft binds to the target's, and the written ones.""" + names = vars(cls).get("_adoption_names_cache") + if names is None: + names = ( + cls._declared_union("draft_bound"), + tuple(sorted(cls._declared_union("draft_written"))), + ) + cls._adoption_names_cache = names + return names + + @classmethod + def draft_unlisted(cls, names) -> FrozenSet[str]: + """The attribute ``names`` an adopting extension lists in none of its + draft lists (bookkeeping of the adoption itself excluded).""" + listed = ( + cls._declared_union("draft_bound") + | cls._declared_union("draft_written") + | cls._declared_union("draft_private") + | _ADOPTION_BOOKKEEPING + ) + return frozenset(names) - listed + + def adopt_target_state(self, source: "MetadataExtension") -> None: + """Take ``source``'s prepared state as this draft's. + + The ``draft_bound`` attributes are bound to the source's, with + dictionaries and sets copied. The ``draft_written`` tensors, and the + tensors of ``draft_written`` dictionaries, keep their own storage and + get the source's contents on device (into a new buffer of the + draft's when the shapes differ). + Everything else stays as it is. + """ + bound, written = type(self)._adoption_names() + state = vars(self) + source_state = vars(source) + # The adopted names change only when the source gains an attribute. + names = state.get("_draft_adopted_names") + if names is None or names[0] != len(source_state): + adopted = tuple(name for name in source_state if name in bound) + containers = tuple( + name for name in adopted if type(source_state[name]) in _COPIED_CONTAINERS + ) + if len(adopted) > 1: + getter = operator.itemgetter(*adopted) + else: + getter = lambda values: tuple(values[name] for name in adopted) # noqa: E731 + names = (len(source_state), adopted, getter, containers) + state["_draft_adopted_names"] = names + _, adopted, getter, containers = names + state.update(zip(adopted, getter(source_state))) + for name in containers: + state[name] = state[name].copy() + sources = tuple(source_state.get(name) for name in written) + mine = tuple(state.get(name) for name in written) + plan = state.get("_draft_copy_plan") + if ( + plan is None + or any(a is not b for a, b in zip(sources, plan[0])) + or any(a is not b for a, b in zip(mine, plan[1])) + ): + plan = self._plan_draft_copies(written, sources, mine) + for destinations, origins in plan[2]: + if destinations[0].dtype in _NO_FOREACH_COPY_DTYPES: + for destination, origin in zip(destinations, origins): + destination.copy_(origin, non_blocking=True) + else: + torch._foreach_copy_(destinations, origins, non_blocking=True) + + def _plan_draft_copies(self, written, sources, mine): + """Group the written tensors into one multi-tensor copy per device and + dtype, first allocating the draft's own where it has none of the + source's shape. + + A written dictionary of tensors is copied entry by entry into a + dictionary of the draft's own.""" + state = vars(self) + groups: Dict[Tuple[Any, Any], Tuple[List[Any], List[Any]]] = {} + + def copyable(own, value) -> bool: + return ( + isinstance(own, torch.Tensor) + and own is not value + and own.shape == value.shape + and own.dtype == value.dtype + and own.device == value.device + ) + + def group(own, value) -> None: + destinations, origins = groups.setdefault((own.device, own.dtype), ([], [])) + destinations.append(own) + origins.append(value) + + for name, value, own in zip(written, sources, mine): + if isinstance(value, dict): + own_entries = own if isinstance(own, dict) and own is not value else {} + entries = {} + for key, entry in value.items(): + own_entry = own_entries.get(key) + if not isinstance(entry, torch.Tensor): + entries[key] = entry + else: + if not copyable(own_entry, entry): + own_entry = torch.empty_like(entry) + entries[key] = own_entry + group(own_entry, entry) + if ( + own_entries is not own + or entries.keys() != own_entries.keys() + or any(entries[key] is not own_entries[key] for key in entries) + ): + state[name] = entries + elif not isinstance(value, torch.Tensor): + state[name] = value + else: + if not copyable(own, value): + own = state[name] = torch.empty_like(value) + group(own, value) + plan = ( + sources, + tuple(state.get(name) for name in written), + tuple(groups.values()), + ) + state["_draft_copy_plan"] = plan + return plan + + def creation_config(self) -> Dict[str, Any]: + """This extension's creation arguments as they stand, for a copy of its metadata.""" + return {name: self.creation_value(name) for name in type(self).declared_init_fields()} + + def creation_value(self, name: str) -> Any: + """The current value of creation argument ``name``; see ``creation_config``.""" + return getattr(self, name) + + def prepare(self) -> None: + """Prepare this extension's per-step state after the core.""" + + def refresh_from_draft_lengths(self) -> None: + """Re-derive this draft extension's state from its changed lengths. + + Called by ``DraftMetadata.refresh_draft()`` on the extensions of + ``md.draft``, in dependency order, after the draft loop changed the + draft's lengths in place and before the next draft forward. Refresh + the state derived from the lengths and reset the previous forward's + state; an override calls ``super()``. + """ + + +def init_field_owner(name: str) -> Optional[type]: + """The extension class that takes the metadata constructor argument ``name``.""" + return _INIT_FIELD_OWNERS.get(name) + + +def reserved_extension_names() -> FrozenSet[str]: + """Names no extension may declare: its own lifecycle and bookkeeping members.""" + return frozenset(name for name in vars(MetadataExtension) if not name.startswith("__")) | { + "metadata" + } + + +def check_extension_names( + metadata: "AttentionMetadata", + cls: Type[MetadataExtension], + attached: Iterable[MetadataExtension], +) -> None: + """Raise when ``cls`` declares a name its metadata or an attached extension owns. + + A core provides its metadata family's state, so it is checked against the + other extensions only. + """ + names = cls.declared_names() + shadowed = set() if cls.is_core else names & metadata.own_attribute_names() + if shadowed: + raise ValueError( + f"{cls.__name__} declares {sorted(shadowed)}, which " + f"{type(metadata).__name__} already defines" + ) + for other in attached: + if issubclass(cls, type(other)) or isinstance(other, cls): + continue + clash = names & type(other).declared_names() + if clash: + raise ValueError( + f"{cls.__name__} and {type(other).__name__} both declare {sorted(clash)}" + ) + + +def resolve_extension_order( + types: Iterable[Type[MetadataExtension]], +) -> List[Type[MetadataExtension]]: + """Order extension types so every requirement precedes its dependents. + + Requirements that are not listed are added. When a base extension and its + subclass are both present, the subclass replaces the base at the base's + dependency position. Listing order is otherwise preserved. + """ + seed = list(types) + all_types: List[Type[MetadataExtension]] = [] + seen: set = set() + + def collect(cls: Type[MetadataExtension]) -> None: + if cls in seen: + return + seen.add(cls) + for required in cls.requires: + collect(required) + all_types.append(cls) + + for cls in seed: + collect(cls) + + replacement: Dict[Type[MetadataExtension], Type[MetadataExtension]] = {} + for cls in all_types: + derived = [other for other in all_types if issubclass(other, cls)] + most_derived = [ + cand + for cand in derived + if all(not issubclass(other, cand) or other is cand for other in derived) + ] + if len(most_derived) > 1: + raise TypeError( + f"Ambiguous subclasses for {cls.__name__}: " + f"{[cand.__name__ for cand in most_derived]}" + ) + replacement[cls] = most_derived[0] + + ordered: List[Type[MetadataExtension]] = [] + visiting: set = set() + + def visit(cls: Type[MetadataExtension]) -> None: + cls = replacement[cls] + if cls in ordered: + return + if cls in visiting: + raise ValueError(f"Metadata extension requirements form a cycle at {cls.__name__}.") + visiting.add(cls) + for required in cls.requires: + target = replacement[required] + if target is not cls: + visit(target) + visiting.discard(cls) + ordered.append(cls) + + for cls in seed: + visit(cls) + return ordered + + +class ExtensionRegistry: + """The extensions attached to one metadata, in dependency order.""" + + def __init__(self, owner: Optional["AttentionMetadata"] = None) -> None: + self.owner = owner + self._by_type: Dict[Type[MetadataExtension], MetadataExtension] = {} + + def __iter__(self): + return iter(self._by_type.values()) + + def __len__(self) -> int: + return len(self._by_type) + + def __contains__(self, cls: Type[MetadataExtension]) -> bool: + return self.find(cls) is not None + + def types(self) -> List[Type[MetadataExtension]]: + return list(self._by_type) + + def core(self) -> Optional[MetadataExtension]: + """The attached core, which is always the first extension, or None.""" + first = next(iter(self._by_type.values()), None) + return first if first is not None and first.is_core else None + + def attach_core(self, core: MetadataExtension) -> None: + """Attach the metadata family's core; it must come before every other extension.""" + if not core.is_core: + raise TypeError(f"{type(core).__name__} is not a core metadata") + if self._by_type: + raise ValueError( + f"The core {type(core).__name__} must be attached before " + f"{[t.__name__ for t in self._by_type]}" + ) + self._by_type[type(core)] = core + + def _require_core(self, cls: Type[MetadataExtension]) -> None: + if cls.is_core: + raise ValueError(f"Attach the core {cls.__name__} with attach_core()") + if self.core() is None: + raise ValueError(f"Attach the core metadata before {cls.__name__}") + + def find(self, cls: Type[E]) -> Optional[E]: + found = self._by_type.get(cls) + if found is not None: + return found + for extension in self._by_type.values(): + if isinstance(extension, cls): + return extension + return None + + def attach( + self, + metadata: "AttentionMetadata", + types: Sequence[Type[MetadataExtension]], + configs: Optional[Dict[type, Dict[str, Any]]] = None, + sources: Optional[Dict[type, MetadataExtension]] = None, + ) -> List[MetadataExtension]: + """Construct and attach the given types and their requirements; return the new ones. + + Each extension is constructed with its routed creation arguments + (``configs``) and, for a copy, the same extension of the source + metadata (``sources``), in dependency order, so its constructor can + read the extensions attached before it. + """ + configs = configs or {} + sources = sources or {} + added = [] + for cls in resolve_extension_order(types): + if cls not in self._by_type: + self._require_core(cls) + check_extension_names(metadata, cls, self._by_type.values()) + # Attached before it is constructed, so its constructor reads + # its own and the earlier extensions' fields through the + # metadata, as every later step does. + extension = cls.__new__(cls) + self._by_type[cls] = extension + metadata._compose() + if cls in sources: + extension.__init__(metadata, source=sources[cls], **configs.get(cls, {})) + else: + extension.__init__(metadata, **configs.get(cls, {})) + extension.__dict__.pop(_COPY_SOURCE, None) + added.append(extension) + return added + + def attach_new_core( + self, + metadata: "AttentionMetadata", + cls: Type[MetadataExtension], + config: Dict[str, Any], + source: Optional[MetadataExtension], + ) -> MetadataExtension: + """Construct and attach the metadata family's core, attached before it is built.""" + core = cls.__new__(cls) + self.attach_core(core) + metadata._compose() + if source is not None: + core.__init__(metadata, source=source, **config) + else: + core.__init__(metadata, **config) + core.__dict__.pop(_COPY_SOURCE, None) + return core + + def replace(self, extension: MetadataExtension) -> None: + """Put an already-built extension in place of any of its type. + + A core replaces the attached core in place, so it stays first. + """ + if extension.is_core: + current = self.core() + if current is None: + self.attach_core(extension) + return + self._by_type = { + (type(extension) if e is current else t): (extension if e is current else e) + for t, e in self._by_type.items() + } + return + if type(extension) not in self._by_type: + self._require_core(type(extension)) + self._by_type[type(extension)] = extension + + def remove(self, cls: Type[MetadataExtension]) -> None: + """Detach the extension of type ``cls``, if any.""" + found = self.find(cls) + if found is not None: + if found.is_core: + raise ValueError(f"The core {type(found).__name__} cannot be detached") + del self._by_type[type(found)] + + def bound_to(self, metadata: "AttentionMetadata") -> "ExtensionRegistry": + """This registry, owned by ``metadata``. + + Returns ``self`` when ``metadata`` already owns it; otherwise a new + registry holding a view of each extension, so a shallow + copy of the metadata never shares the registry or its extensions with + its source. + """ + if self.owner is metadata: + return self + clone = ExtensionRegistry(metadata) + clone._by_type = {cls: _view_of(e, metadata) for cls, e in self._by_type.items()} + return clone + + def with_core_copied(self, metadata: "AttentionMetadata") -> "ExtensionRegistry": + """A registry for a ``copy.copy`` of the owner: its own core, shared other extensions. + + The copy owns its core state, as it owned the core's fields before they + moved into the core; the other extensions stay shared until the copy + binds its own registry (``bound_to``), as before. + """ + clone = ExtensionRegistry(self.owner) + clone._by_type = { + cls: (_view_of(e, metadata) if e.is_core else e) for cls, e in self._by_type.items() + } + return clone + + +def _view_of(extension: MetadataExtension, metadata: "AttentionMetadata") -> MetadataExtension: + """A shallow view of ``extension`` bound to ``metadata`` (a ``copy.copy`` view). + + Views share every attribute with their source; a CUDA graph or + cross-attention copy is constructed instead (see ``shared``). + """ + view = copy.copy(extension) + view.metadata = metadata + return view diff --git a/tensorrt_llm/_torch/attention/backends/features/__init__.py b/tensorrt_llm/_torch/attention/backends/features/__init__.py new file mode 100644 index 000000000000..130baa9ed7f7 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Feature extensions of the attention metadata; see ``..extensions``.""" + +from .cross import CrossAttentionMetadata +from .draft import DraftMetadata +from .flash_mla import FlashMlaMetadata +from .helix import HelixMetadata +from .recurrent import RecurrentConfig, RecurrentMetadata + +__all__ = [ + "CrossAttentionMetadata", + "DraftMetadata", + "FlashMlaMetadata", + "HelixMetadata", + "RecurrentConfig", + "RecurrentMetadata", +] diff --git a/tensorrt_llm/_torch/attention/backends/features/cross.py b/tensorrt_llm/_torch/attention/backends/features/cross.py new file mode 100644 index 000000000000..096198f1ee0c --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/cross.py @@ -0,0 +1,247 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cross-attention state of an encoder-decoder model's attention metadata.""" + +from typing import TYPE_CHECKING, Any, List, Optional + +import torch + +from ....memory_buffer_utils import Buffers +from ....metadata import KVCacheParams +from ..extensions import MetadataExtension + +if TYPE_CHECKING: + from ..interface import AttentionMetadata, BatchInputs + + +class CrossAttentionMetadata(MetadataExtension): + """The cross-attention sub-metadata of an encoder-decoder model. + + On a decoder's self-attention metadata, ``cross`` is the sub-metadata its + cross-attention layers use: it shares the decoder's query side and takes + the encoder's lengths as its key/value side. On that sub-metadata, which + is cross attention itself (``is_cross``), ``cross`` refers to itself. + """ + + fields = frozenset({"cross", "has_sub_metadata"}) + methods = frozenset( + { + "update_cross_metadata", + "refresh_cross_query_side", + "use_cross_sub_metadata", + "drop_cross_sub_metadata", + } + ) + init_fields = { + "cross": None, + } + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + md = self.metadata + if md.is_cross: + assert self.cross is None or self.cross is md, ( + "Cross attention metadata should not have sub metadata" + ) + self.cross = md + else: + assert self.cross is None or (self.cross.core_class() is md.core_class()), ( + "Top level and cross attention sub metadata type mismatched" + ) + + @property + def has_sub_metadata(self) -> bool: + return self.cross is not None and self.cross is not self.metadata + + def creation_value(self, name: str) -> Any: + # The sub-metadata refers to itself; a copy of it refers to the copy. + if name == "cross" and self.cross is self.metadata: + return None + return super().creation_value(name) + + def use_cross_sub_metadata(self, cross_md: "AttentionMetadata") -> None: + """Keep ``cross_md`` as this decoder's sub-metadata (CUDA graph steps reuse it).""" + self.cross = cross_md + + def drop_cross_sub_metadata(self) -> None: + """Detach the sub-metadata from a derived view (a draft view) of the decoder.""" + if self.has_sub_metadata: + self.cross = None + + def copy_sub_metadata_for_cuda_graph( + self, max_batch_size: int, max_draft_tokens: int, buffers, encode_only: bool + ) -> None: + """Point a CUDA graph copy at a CUDA graph copy of the sub-metadata.""" + if self.has_sub_metadata: + self.cross = self.cross.create_cuda_graph_metadata( + max_batch_size, True, max_draft_tokens, buffers, encode_only + ) + + def create_sub_metadata( + self, + encoder_seq_lens: List[int], + cross_kv_cache_manager=None, + encoder_num_cached_tokens_per_seq: Optional[List[int]] = None, + ) -> "AttentionMetadata": + """Build a sub-metadata instance for cross-attention. + + The returned metadata shares Q-side fields (``seq_lens``, + ``request_ids``, ``num_contexts``) with ``self`` (the decoder + self-attention metadata) and overrides the K/V-side with the encoder + lengths so that ``returned.is_cross is True``. + + This is intended to be called by the runtime / unit tests once the + encoder lengths and cross-pool KV cache manager are known. The + returned object is a *new* metadata instance (not stored on + the decoder's ``cross``); callers can attach it there if desired. + + Args: + encoder_seq_lens: Per-request encoder sequence lengths. On the + first decoder context step this is the full encoder length; + on generation steps it should be ``0`` (no new K/V tokens to + add to the cross pool — the encoder K/V are already cached). + cross_kv_cache_manager: KV cache manager for the cross pool. + When ``None``, the returned metadata uses the stateless + (no-KV-cache) path (suitable for unit tests). + encoder_num_cached_tokens_per_seq: Per-request count of encoder + K/V tokens already present in the cross pool. ``None`` + defaults to 0 (context phase, nothing cached yet). + + Returns: + A new ``AttentionMetadata`` of the same family as the decoder's, + with ``seq_lens_kv`` set to ``encoder_seq_lens`` so that + ``is_cross`` becomes ``True``. + """ + md = self.metadata + # Cross-attention has K/V lengths from the encoder, while + # self-attention has K/V lengths from the decoder. In a CUDA graph the + # sub-metadata keeps its own buffers so preparing cross metadata cannot + # overwrite self-attention sequence lengths. + cross_md = md.constructed_copy( + kv_cache_manager=cross_kv_cache_manager, + seq_lens_kv=torch.tensor(encoder_seq_lens, dtype=torch.int), + cross=None, + cuda_graph_buffers=Buffers() if md.is_cuda_graph else md.cuda_graph_buffers, + ) + self._update_sub_metadata( + cross_md, + encoder_seq_lens, + cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq, + base_kv_cache_params=md.kv_cache_params, + block_ids_per_seq=None, + ) + return cross_md + + def _update_sub_metadata( + self, + cross_md: "AttentionMetadata", + encoder_seq_lens: List[int], + cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq: Optional[List[int]], + *, + base_kv_cache_params: Optional[KVCacheParams], + block_ids_per_seq: Optional[List[list]], + ) -> "AttentionMetadata": + kv_cache_params = cross_md.kv_cache_params + if encoder_num_cached_tokens_per_seq is not None: + kv_cache_params = KVCacheParams( + use_cache=( + base_kv_cache_params.use_cache + if base_kv_cache_params is not None + else (cross_kv_cache_manager is not None) + ), + num_cached_tokens_per_seq=list(encoder_num_cached_tokens_per_seq), + block_ids_per_seq=block_ids_per_seq, + host_max_attention_window_sizes=( + base_kv_cache_params.host_max_attention_window_sizes + if base_kv_cache_params is not None + else None + ), + host_sink_token_length=( + base_kv_cache_params.host_sink_token_length + if base_kv_cache_params is not None + else None + ), + num_extra_kv_tokens=( + base_kv_cache_params.num_extra_kv_tokens + if base_kv_cache_params is not None + else 0 + ), + ) + # The sub-metadata shares the decoder's query side, takes the encoder + # lengths as its KV side and keeps the decoder-side prompt lengths for + # the Q-side context metadata (the encoder side is in seq_lens_kv and + # kv_cache_params.num_cached_tokens_per_seq). + cross_md.core().kv_cache_manager = cross_kv_cache_manager + cross_md.core().apply_batch_inputs( + self._sub_metadata_inputs( + cross_md, + kv_cache_params=kv_cache_params, + seq_lens_kv=torch.tensor(encoder_seq_lens, dtype=torch.int), + ) + ) + return cross_md + + def _sub_metadata_inputs(self, cross_md: "AttentionMetadata", **changes) -> "BatchInputs": + """The sub-metadata's batch: the decoder's query side, its own otherwise.""" + from ..interface import BatchInputs + + md = self.metadata + inputs = dict( + seq_lens=md.seq_lens, + seq_lens_cuda=md.seq_lens_cuda, + num_contexts=md.num_contexts, + request_ids=md.request_ids, + prompt_lens=md.prompt_lens, + kv_cache_params=cross_md.kv_cache_params, + beam_width=cross_md.beam_width, + num_chunked_ctx_requests=cross_md.num_chunked_ctx_requests, + q_offsets=cross_md.cu_q_seqlens, + kv_offsets=cross_md.cu_kv_seqlens, + ) + inputs.update(changes) + return BatchInputs(**inputs) + + def refresh_cross_query_side(self) -> "AttentionMetadata": + """Rebind the sub-metadata to the decoder's current query side. + + For a stable CUDA graph generation step whose encoder side did not + change: binds the decoder's lengths, request IDs and prompt lengths + without staging or re-deriving anything. + """ + if not self.has_sub_metadata: + raise RuntimeError("CUDA graph cross-attention metadata has not been initialized.") + self.cross.core().apply_batch_inputs(self._sub_metadata_inputs(self.cross)) + return self.cross + + def prepare_sub_metadata(self, inputs: "BatchInputs") -> None: + """Prepare the sub-metadata from its own step inputs (``BatchInputs.cross``).""" + if not self.has_sub_metadata: + raise RuntimeError("BatchInputs.cross given without a cross-attention sub-metadata.") + self.cross.prepare(inputs) + + def update_cross_metadata( + self, + encoder_seq_lens: List[int], + cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq: Optional[List[int]] = None, + ) -> "AttentionMetadata": + """Refresh an existing CUDA graph cross-attention sub-metadata.""" + if not self.has_sub_metadata: + raise RuntimeError("CUDA graph cross-attention metadata has not been initialized.") + + cross_md = self.cross + assert cross_md is not None + base_kv_cache_params = cross_md.kv_cache_params + block_ids_per_seq = ( + base_kv_cache_params.block_ids_per_seq if base_kv_cache_params is not None else None + ) + return self._update_sub_metadata( + cross_md, + encoder_seq_lens, + cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq, + base_kv_cache_params=base_kv_cache_params, + block_ids_per_seq=block_ids_per_seq, + ) diff --git a/tensorrt_llm/_torch/attention/backends/features/draft.py b/tensorrt_llm/_torch/attention/backends/features/draft.py new file mode 100644 index 000000000000..1063840cb8db --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/draft.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The draft model's attention metadata of one-model speculative decoding.""" + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Dict, Iterator, Optional + +from ....memory_buffer_utils import Buffers +from ..extensions import MetadataExtension, init_field_owner + +if TYPE_CHECKING: + from ..interface import AttentionMetadata + +# Creation settings every draft metadata gets, each given only when the +# extension that takes it is attached: a draft builds no decoder replay +# metadata and no mixed-precision high tables of DeepSeek-V4's MTP layers, and +# its mixed-precision staging copy has buffer names of its own. +_DRAFT_SETTINGS: Dict[str, Any] = { + "builds_decoder_replay": False, + "builds_mixed_precision_high_tables": False, + "mixed_precision_staging_name": "draft_mixed", +} + + +class DraftMetadata(MetadataExtension): + """The draft forwards' own metadata, ``md.draft``, and the draft loop. + + One-model speculative decoding runs the draft forwards on a metadata of + their own: a copy of the target's (all of its extensions but this one and + cross attention) over the draft KV cache manager, or the target's when the + cache is shared. ``md.draft`` is an ordinary metadata: it holds no + reference to the target. Everything that needs both sides is here. + + - Which state the draft takes from the target is fixed at creation. Over + the target's KV cache, when every extension adopts the target's state, + the draft adopts it at the start of each draft loop (inside the CUDA + graph that records the loop) and is never prepared. Any other draft, + such as one over a separate draft KV cache (whose core then stages the + whole block table), is prepared in the target's ``prepare`` from the same + batch, and takes the target's lengths and the state of the extensions + that adopt it always (speculative decoding) at the loop start. + - A draft loop runs inside ``with md.draft_loop() as draft``. It changes + only the draft's lengths, through its core's length primitives, and + calls ``md.refresh_draft()`` before each draft forward, which + re-derives the draft's length-dependent state when they changed. + ``refresh_draft`` raises outside a draft loop. + + A CUDA graph copy of the target builds a graph copy of the draft the same + way, over a buffer pool every graph copy of this draft shares. + """ + + fields = frozenset( + { + "_draft_adopts", + "draft", + "draft_or_none", + "one_model_draft", + "draft_kv_cache_manager", + "in_draft_loop", + "draft_adopts_target_state", + } + ) + methods = frozenset({"draft_loop", "refresh_draft"}) + init_fields = { + # Whether the draft forwards run in this engine and get a metadata of + # their own (one-model speculative decoding). + "one_model_draft": False, + # The draft model's KV cache manager when the draft keeps a separate + # KV cache; None shares the target's. + "draft_kv_cache_manager": None, + } + # Its draft is a copy of the complete target metadata. + constructed_last = True + # The draft's CUDA graph copies share one pool, apart from the target's. + shared = frozenset({"_draft_graph_pool"}) + # A draft's own copy of this extension is inert: it has no draft of its + # own, so it never stops its draft from adopting the target's state. + adopts_target_state = True + draft_private = frozenset( + { + "one_model_draft", + "draft_kv_cache_manager", + "_draft", + "_draft_adopts", + "_in_draft_loop", + "_draft_graph_pool", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self._draft: Optional["AttentionMetadata"] = None + self._draft_adopts = False + self._in_draft_loop = False + # A copy of this metadata takes the source's pool, so every CUDA graph + # copy's draft allocates from the same one. + self._draft_graph_pool = self.shared_or("_draft_graph_pool", Buffers) + if self.one_model_draft: + self._draft = self._build_draft() + self._draft_adopts = ( + not self._separate_draft_cache() and self._draft.adopts_target_state() + ) + + @property + def draft(self) -> "AttentionMetadata": + """The draft forwards' metadata.""" + if self._draft is None: + raise RuntimeError("This metadata has no one-model draft metadata") + return self._draft + + @property + def draft_or_none(self) -> Optional["AttentionMetadata"]: + """The draft forwards' metadata, or None without one-model drafting.""" + return self._draft + + @property + def draft_adopts_target_state(self) -> bool: + """Whether the draft takes the target's prepared state at each draft-loop + start (shared KV cache, every extension adopting) instead of being + prepared with the target.""" + return self._draft_adopts + + @property + def in_draft_loop(self) -> bool: + """Whether a ``draft_loop()`` is running.""" + return self._in_draft_loop + + def _separate_draft_cache(self) -> bool: + manager = self.draft_kv_cache_manager + return manager is not None and manager is not self.metadata.kv_cache_manager + + def _build_draft(self) -> "AttentionMetadata": + """The draft metadata: a copy of the complete target metadata. + + The draft KV cache manager (or the target's when the cache is shared), + the target's extensions but this one and cross attention, and buffers + of its own: from the target's pool when eager, from the shared draft + pool in a CUDA graph. + """ + from .cross import CrossAttentionMetadata + + md = self.metadata + pool = self._draft_graph_pool if md.is_cuda_graph else md.cuda_graph_buffers + separate = self._separate_draft_cache() + settings = { + name: value + for name, value in _DRAFT_SETTINGS.items() + if (owner := init_field_owner(name)) is not None and md.find_ext(owner) is not None + } + return md.constructed_copy( + share=False, + without=(DraftMetadata, CrossAttentionMetadata), + kv_cache_manager=self.draft_kv_cache_manager if separate else md.kv_cache_manager, + cuda_graph_buffers=pool, + # Draft steps advance the device KV lengths past the host + # snapshot, so a draft over its own cache stages the whole table. + full_block_table_width=separate or md.full_block_table_width, + # Its lengths come from its first prepare: a CUDA graph target is + # built before its graph-sized lengths are bound. + seq_lens=None, + seq_lens_kv=None, + **settings, + ) + + def prepare(self) -> None: + """Prepare a draft that derives its own state from the target's batch. + + A draft that adopts the target's state does so at the start of its + draft loop instead. After a steady decode step of the target the draft + takes its own steady step. + """ + draft = self._draft + if draft is None or self._draft_adopts: + return + from ..interface import BatchInputs + + md = self.metadata + inputs = BatchInputs( + seq_lens=md.seq_lens.clone() if md.seq_lens is not None else None, + num_contexts=md.num_contexts, + request_ids=md.request_ids, + prompt_lens=md.prompt_lens, + kv_cache_params=md.kv_cache_params, + beam_width=md.beam_width, + num_chunked_ctx_requests=md.num_chunked_ctx_requests, + kv_lens_offsets=md.kv_lens_offsets, + position_id_offsets=md.position_id_offsets, + ) + kv_lens = md.steady_decode_kv_lens + if kv_lens is not None: + draft.prepare_steady_decode_step(inputs, kv_lens) + else: + draft.prepare(inputs) + + @contextmanager + def draft_loop(self) -> Iterator["AttentionMetadata"]: + """Run one draft loop on ``md.draft``, started from the target's state. + + An adopting draft takes the target's prepared state (``adopt_target_state`` + of each extension, core first); any other draft takes the target's lengths + and counts (``copy_lengths_from``, which also brings the engine's + attention-DP token counts) and the state of the extensions that adopt it + always. Both run inside a captured graph, so every replay starts from the + target. Yields the draft; the loop scope ends with the block. + """ + draft = self.draft + target = self.metadata + core = draft.core() + if self._draft_adopts: + for extension in draft._attached_extensions(): + if isinstance(extension, DraftMetadata): + continue + source = target.core() if extension.is_core else target.find_ext(type(extension)) + extension.adopt_target_state(source) + else: + core.copy_lengths_from(target.core()) + for extension in draft._attached_extensions(): + if type(extension).adopts_target_state_always: + extension.adopt_target_state(target.find_ext(type(extension))) + core.clear_lengths_changed() + self._in_draft_loop = True + try: + yield draft + finally: + self._in_draft_loop = False + + def refresh_draft(self) -> None: + """Re-derive ``md.draft``'s length-dependent state after the draft loop + changed its lengths in place; nothing when they did not change. + + The core re-derives its eagerly derived state + (``resync_length_derived_state``) and every other extension refreshes + from the draft's current lengths (``refresh_from_draft_lengths``), in + dependency order. Raises outside ``draft_loop()``. + """ + if not self._in_draft_loop: + raise RuntimeError("refresh_draft() runs inside md.draft_loop() only") + draft = self._draft + core = draft.core() + if not core.lengths_changed: + return + core.resync_length_derived_state() + for extension in draft._attached_extensions(): + if extension is not core: + extension.refresh_from_draft_lengths() + core.clear_lengths_changed() diff --git a/tensorrt_llm/_torch/attention/backends/features/flash_mla.py b/tensorrt_llm/_torch/attention/backends/features/flash_mla.py new file mode 100644 index 000000000000..b7905d1fd6b4 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/flash_mla.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""FlashMLA state of the attention metadata.""" + +from typing import Optional + +import torch + +from tensorrt_llm._utils import maybe_pin_memory +from tensorrt_llm.bindings.internal import thop + +from ..extensions import MetadataExtension + + +class FlashMlaMetadata(MetadataExtension): + """Block IDs and tile-scheduler metadata of the FlashMLA generation kernels. + + Provides the core's ``block_ids_per_seq`` (generation rows, compacted) and + ``kv_block_ids_per_seq`` (every row). The tile-scheduler metadata is + computed once per forward by the first generation layer and reused until + the KV lengths or block IDs change. + """ + + fields = frozenset( + { + "metadata_valid", + "flash_mla_num_splits", + "flash_mla_tile_scheduler_metadata", + } + ) + methods = frozenset( + { + "compute_scheduler_metadata", + "prepare_block_ids", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.flash_mla_tile_scheduler_metadata: Optional[torch.Tensor] = None + self.flash_mla_num_splits: Optional[torch.Tensor] = None + self.metadata_valid = False + buffers = metadata.cuda_graph_buffers + md = self.metadata + manager = md.kv_cache_manager + if manager is None: + return + capture_graph = md.is_cuda_graph + + def block_ids(owner, name): + return md.get_empty( + buffers, + [owner.max_batch_size, owner.max_blocks_per_seq], + cache_name=name, + dtype=torch.int32, + capture_graph=capture_graph, + ) + + md.use_block_id_tables( + block_ids(manager, "block_ids_per_seq"), block_ids(manager, "kv_block_ids_per_seq") + ) + # Pre-allocated so their GPU addresses are stable across CUDA graph captures. + sm_count = torch.cuda.get_device_properties( + torch.cuda.current_device() + ).multi_processor_count + self.flash_mla_tile_scheduler_metadata = md.get_empty( + buffers, + [sm_count * 8], # TileSchedulerMetaDataSize = 8 + cache_name="flash_mla_tile_scheduler_metadata", + dtype=torch.int32, + capture_graph=capture_graph, + ) + self.flash_mla_num_splits = md.get_empty( + buffers, + [manager.max_batch_size + 1], + cache_name="flash_mla_num_splits", + dtype=torch.int32, + capture_graph=capture_graph, + ) + + adopts_target_state = True + # Its tile schedule, and whether it is current, are computed lazily from + # a draft's own lengths. + draft_private = frozenset( + {"flash_mla_tile_scheduler_metadata", "flash_mla_num_splits", "metadata_valid"} + ) + + def prepare(self) -> None: + self.prepare_block_ids() + + def adopt_target_state(self, source: "MetadataExtension") -> None: + super().adopt_target_state(source) + # Its tile schedule follows its own lengths. + self.metadata_valid = False + + def prepare_block_ids(self) -> None: + """Stage this batch's block IDs from the current KV cache manager.""" + md = self.metadata + self.metadata_valid = False + # Request-specific fills and H2D copies must happen before replay, not + # become fixed operations in the captured graph. + if torch.cuda.is_current_stream_capturing(): + return + + block_ids_per_seq = maybe_pin_memory( + md.kv_cache_manager.get_block_ids_per_seq(md.request_ids) + ) + num_blocks = block_ids_per_seq.shape[1] + md.kv_block_ids_per_seq.fill_(0) + md.kv_block_ids_per_seq[: md.num_seqs, :num_blocks].copy_( + block_ids_per_seq, non_blocking=True + ) + md.block_ids_per_seq.fill_(0) + md.block_ids_per_seq[: md.num_generations, :num_blocks].copy_( + block_ids_per_seq[md.num_contexts :], non_blocking=True + ) + + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + self.metadata_valid = False + + def compute_scheduler_metadata(self, num_heads: int, kv_lora_rank: int) -> None: + """Compute the tile-scheduler metadata once per forward.""" + md = self.metadata + if self.metadata_valid: + return + num_generations = md.num_generations + if num_generations > 0: + generation_slice = slice(md.num_contexts, md.num_contexts + num_generations) + s_q = int(md.seq_lens[md.num_contexts].item()) + thop.compute_flash_mla_metadata( + md.kv_lens_cuda_runtime[generation_slice], + self.flash_mla_tile_scheduler_metadata, + self.flash_mla_num_splits[: num_generations + 1], + num_generations, + s_q, + num_heads, + 1, + kv_lora_rank, + ) + self.metadata_valid = True diff --git a/tensorrt_llm/_torch/attention/backends/features/helix.py b/tensorrt_llm/_torch/attention/backends/features/helix.py new file mode 100644 index 000000000000..6aa1a2ed7500 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/helix.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Helix context-parallel state of the attention metadata.""" + +from typing import List, Optional + +import torch + +from tensorrt_llm._utils import prefer_pinned + +from ....utils import helix_local_len_tensor +from ..extensions import MetadataExtension +from ..fp4_mla.metadata import populate_helix_spec_buffers + + +class HelixMetadata(MetadataExtension): + """Per-token global positions and KV ownership under helix parallelism. + + Each helix rank holds a subset of a sequence's tokens. Only the active rank + appends the KV of a new token and attends to it; inactive ranks attend to + the previously cached tokens only. + + The per-sequence and per-token buffers are GENERATION-relative: the + packing loops in the executor append only for extend and plain generation + rows, so ``update_helix_param`` writes exactly ``[0, num_generations)``. + """ + + fields = frozenset( + { + "helix_position_offsets", + "helix_position_offsets_cpu", + "helix_is_inactive_rank", + "helix_is_inactive_rank_cpu", + "helix_kv_bounds", + "helix_local_slots", + "helix_owned_new_tokens_cpu", + "_helix_spec_tokens_valid", + } + ) + methods = frozenset( + { + "helix_local_len_vec", + "recompute_helix_spec_buffers", + "update_helix_param", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Global position id of each token of each sequence. + self.helix_position_offsets: Optional[torch.Tensor] = None + self.helix_position_offsets_cpu: Optional[torch.Tensor] = None + # Whether this rank is inactive for each generation sequence. + self.helix_is_inactive_rank: Optional[torch.Tensor] = None + self.helix_is_inactive_rank_cpu: Optional[torch.Tensor] = None + # Per-token state for speculative verify groups: a 1 + draft_len group + # may straddle a ledger-page boundary onto two CP ranks, so the + # per-sequence flag above is not expressive enough. See + # recompute_helix_spec_buffers for the derivation. + # helix_local_slots[t]: rank-local KV write slot of generation token t on + # this rank, or -1 when another rank owns its position (consumed + # by the mla_rope_generation append kernel). + # helix_kv_bounds[t]: number of rank-local KV entries token t may attend + # to, i.e. local_len(pos_t + 1) (consumed by the CuTe DSL MLA + # decode mask and the helix stats identity). + self.helix_local_slots: Optional[torch.Tensor] = None + self.helix_kv_bounds: Optional[torch.Tensor] = None + # Host per-sequence count of this step's new tokens owned by this rank + # (spec path; the single-token path derives it from the boolean flag). + self.helix_owned_new_tokens_cpu: Optional[torch.Tensor] = None + self._helix_spec_tokens_valid = False + # Width of this step's speculative verify groups (spec path). + self._tokens_per_gen_seq = 0 + buffers = metadata.cuda_graph_buffers + md = self.metadata + capture_graph = md.is_cuda_graph + self.helix_position_offsets = md.get_empty( + buffers, + (md.max_num_tokens,), + cache_name="helix_position_offsets", + dtype=torch.int, + capture_graph=capture_graph, + ) + self.helix_position_offsets_cpu = torch.empty_like( + self.helix_position_offsets, + device="cpu", + pin_memory=prefer_pinned(), + ) + self.helix_is_inactive_rank = md.get_empty( + buffers, + (md.max_num_sequences,), + cache_name="helix_is_inactive_rank", + dtype=torch.bool, + capture_graph=capture_graph, + ) + self.helix_is_inactive_rank_cpu = torch.empty_like( + self.helix_is_inactive_rank, + device="cpu", + pin_memory=prefer_pinned(), + ) + self.helix_local_slots = md.get_empty( + buffers, + (md.max_num_tokens,), + cache_name="helix_local_slots", + dtype=torch.int, + capture_graph=capture_graph, + ) + self.helix_kv_bounds = md.get_empty( + buffers, + (md.max_num_tokens,), + cache_name="helix_kv_bounds", + dtype=torch.int, + capture_graph=capture_graph, + ) + self.helix_owned_new_tokens_cpu = torch.zeros( + (md.max_num_sequences,), + device="cpu", + dtype=torch.int, + pin_memory=prefer_pinned(), + ) + self._helix_spec_tokens_valid = False + + def update_helix_param( + self, + helix_position_offsets: List[int], + helix_is_inactive_rank: List[bool], + helix_owned_new_tokens: Optional[List[int]] = None, + tokens_per_gen_seq: int = 1, + ) -> None: + """Copy this step's helix parameters into the static buffers. + + Args: + helix_position_offsets: Global position of each token, shape (num_tokens,). + helix_is_inactive_rank: Whether this rank is inactive for each + generation sequence, shape (num_generations,). + helix_owned_new_tokens: Per-sequence count of this step's new tokens + owned by this rank (speculative verify groups; one group may + straddle a page boundary onto two ranks). None on the + single-token path, where the boolean flag carries it. + tokens_per_gen_seq: Width of the speculative verify groups; read + only with ``helix_owned_new_tokens``. + """ + if helix_position_offsets is not None: + num_tokens = len(helix_position_offsets) + self.helix_position_offsets_cpu[:num_tokens].copy_( + torch.tensor(helix_position_offsets, dtype=torch.int) + ) + self.helix_position_offsets[:num_tokens].copy_( + self.helix_position_offsets_cpu[:num_tokens], non_blocking=True + ) + + if helix_is_inactive_rank is not None: + batch_size = len(helix_is_inactive_rank) + self.helix_is_inactive_rank_cpu[:batch_size].copy_( + torch.tensor(helix_is_inactive_rank, dtype=torch.bool) + ) + self.helix_is_inactive_rank[:batch_size].copy_( + self.helix_is_inactive_rank_cpu[:batch_size], non_blocking=True + ) + + self._helix_spec_tokens_valid = False + if helix_owned_new_tokens is not None: + batch_size = len(helix_owned_new_tokens) + self.helix_owned_new_tokens_cpu[:batch_size].copy_( + torch.tensor(helix_owned_new_tokens, dtype=torch.int) + ) + self._helix_spec_tokens_valid = True + self._tokens_per_gen_seq = tokens_per_gen_seq + + def prepare(self) -> None: + """Count only the new tokens this rank appends for generation rows. + + Context rows are not part of a verify group and are not packed into + the helix buffers; they append every one of their tokens. Generation + rows pair the generation-relative helix buffers with the generation + slice of the batch-indexed lengths. The core republishes the corrected + KV lengths before the extensions after this one read them. + + With speculative verify groups the device KV lengths are then derived + on device from the corrected positions (``recompute_helix_spec_buffers``). + This override supersedes the core's ``kv_lens_offsets`` correction, + which is not ownership-aware. + """ + md = self.metadata + assert md.kv_cache_params.use_cache, "helix requires a KV cache" + kv_lens = md.kv_lens_runtime.clone() + num_gen = md.num_generations + gen = slice(md.num_contexts, md.num_seqs) + cached = kv_lens[gen] - md.seq_lens_kv[gen] + if self._helix_spec_tokens_valid: + # Speculative verify groups: ownership of this step's new tokens is + # a per-sequence COUNT, not a boolean. Provisional host values, + # which the device derivation below overrides. + kv_lens[gen] = cached + self.helix_owned_new_tokens_cpu[:num_gen] + else: + kv_lens[gen] = torch.where( + self.helix_is_inactive_rank_cpu[:num_gen], cached, kv_lens[gen] + ) + md.republish_kv_lens(kv_lens) + if self._helix_spec_tokens_valid and md.kv_cache_manager is not None: + num_gen_tokens = md.num_tokens - md.num_ctx_tokens + if md.position_id_offsets is not None: + # The host packed provisional positions from a stale base; apply + # the overlap scheduler's device correction. Staged again from + # the host values so that a repeated prepare does not apply it + # twice. + positions = self.helix_position_offsets[:num_gen_tokens] + positions.copy_(self.helix_position_offsets_cpu[:num_gen_tokens], non_blocking=True) + positions += md.position_id_offsets[:num_gen_tokens] + self.recompute_helix_spec_buffers(num_gen_tokens, self._tokens_per_gen_seq) + + def helix_local_len_vec(self, global_lens: torch.Tensor) -> torch.Tensor: + """Vectorized rank-local prefix length for helix round-robin pages. + + For each global sequence length g, returns the number of the first g + tokens whose ledger page lives on this CP rank (page b -> rank + b % cp_size). The rule and its scalar twin live in ``_torch.utils``; + KVCacheManagerV2._helix_local_len and the host packing in the executor + use the same definition. + """ + md = self.metadata + return helix_local_len_tensor( + global_lens, + md.kv_cache_manager.tokens_per_block, + md.mapping.cp_size, + md.mapping.cp_rank, + ) + + def recompute_helix_spec_buffers(self, num_gen_tokens: int, tokens_per_gen_seq: int) -> None: + """Derive per-token helix buffers from (corrected) global positions. + + Called after the overlap-scheduler device correction has been applied + to position_offsets, so every derived quantity reflects the real + committed length even though the host packed provisional values. + The generation rows' device KV lengths become the rank-local lengths. + + ``tokens_per_gen_seq`` is the uniform verify-group width. Non-uniform + groups are rejected rather than silently mis-sliced: without the + overlap scheduler the extend loop packs a per-request + ``1 + get_draft_token_length(request)`` and a request entering with no + draft tokens is packed as a single-token generation row instead, so a + batch can arrive whose total happens to divide but whose rows do not + line up. + + Two index bases meet here, and they are not the same: + * helix_position_offsets / helix_local_slots / helix_kv_bounds are GENERATION-RELATIVE. + * kv_lens_cuda is BATCH-indexed, hence the num_contexts offset on + the write below. + """ + md = self.metadata + assert num_gen_tokens % tokens_per_gen_seq == 0, ( + f"helix spec expects uniform verify groups: {num_gen_tokens} gen " + f"tokens not divisible by group size {tokens_per_gen_seq}" + ) + num_gen_seqs = num_gen_tokens // tokens_per_gen_seq + # Divisibility alone does not imply uniformity: a batch of mixed group + # widths can still divide and would then write the wrong number of + # kv_lens_cuda rows with values taken from the wrong tokens. + assert num_gen_seqs == md.num_generations, ( + f"helix spec expects uniform verify groups: {num_gen_tokens} gen " + f"tokens over {md.num_generations} generation rows do not all " + f"have width {tokens_per_gen_seq}" + ) + # One launch derives the write slots, the attention bounds and the + # per-sequence rank-local kv length (the bound of the sequence's last + # token: committed + owned in-flight tokens) from the positions; it + # replaces a chain of elementwise ops on the step's critical path. + local_kv_lens = torch.empty( + (num_gen_seqs,), dtype=torch.int32, device=self.helix_position_offsets.device + ) + populate_helix_spec_buffers( + self.helix_position_offsets[:num_gen_tokens], + self.helix_local_slots[:num_gen_tokens], + self.helix_kv_bounds[:num_gen_tokens], + local_kv_lens, + num_tokens=num_gen_tokens, + tokens_per_seq=tokens_per_gen_seq, + tokens_per_block=md.kv_cache_manager.tokens_per_block, + cp_size=md.mapping.cp_size, + cp_rank=md.mapping.cp_rank, + ) + md.set_kv_lens(slice(md.num_contexts, md.num_contexts + num_gen_seqs), local_kv_lens) diff --git a/tensorrt_llm/_torch/attention/backends/features/recurrent.py b/tensorrt_llm/_torch/attention/backends/features/recurrent.py new file mode 100644 index 000000000000..173d89aa1004 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/recurrent.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Recurrent-layer state of the attention metadata of a hybrid model.""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Type + +from ..extensions import MetadataExtension + +if TYPE_CHECKING: + from ....modules.mamba.mamba2_metadata import Mamba2Metadata + + +@dataclass(frozen=True) +class RecurrentConfig: + """Construction-time configuration of ``RecurrentMetadata``. + + ``chunk_size`` is the model's SSM chunk size; ``mamba_metadata_cls`` is the + ``Mamba2Metadata`` subclass the model needs, or None for the base class. + """ + + chunk_size: int = 128 + mamba_metadata_cls: Optional[Type["Mamba2Metadata"]] = None + + +class RecurrentMetadata(MetadataExtension): + """Per-step metadata of the Mamba and linear-attention layers. + + Attached when the KV cache manager also holds recurrent state. Its chunk + size and ``Mamba2Metadata`` class come from ``recurrent_config``, given at + construction; the instance is built on the first ``prepare()``. + """ + + # Every CUDA graph copy shares the recurrent-state metadata the first + # prepare built. + shared = frozenset({"mamba_metadata"}) + fields = frozenset( + { + "recurrent_config", + "chunk_size", + "mamba_metadata", + "mamba_metadata_cls", + } + ) + methods = frozenset() + init_fields = { + "recurrent_config": RecurrentConfig(), + } + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.mamba_metadata: Optional["Mamba2Metadata"] = self.shared_or( + "mamba_metadata", lambda: None + ) + + @property + def chunk_size(self) -> int: + return self.recurrent_config.chunk_size + + @property + def mamba_metadata_cls(self) -> Optional[Type["Mamba2Metadata"]]: + return self.recurrent_config.mamba_metadata_cls + + def prepare(self) -> None: + md = self.metadata + if self.mamba_metadata is None: + metadata_cls = self.mamba_metadata_cls + if metadata_cls is None: + from ....modules.mamba.mamba2_metadata import Mamba2Metadata + + metadata_cls = Mamba2Metadata + self.mamba_metadata = metadata_cls( + md.max_num_requests, self.chunk_size, max_num_tokens=md.max_num_tokens + ) + self.mamba_metadata.prepare(md) diff --git a/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py b/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py new file mode 100644 index 000000000000..ed9fb1f5022f --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py @@ -0,0 +1,422 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Speculative-decoding (multi-query generation) state of the TRTLLM metadata.""" + +import functools +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +import torch + +from tensorrt_llm._utils import get_sm_version + +from ..extensions import MetadataExtension + +if TYPE_CHECKING: + from ....speculative.interface import SpecMetadata + from ....speculative.spec_tree_manager import SpecTreeManager + + +def is_trtllm_gen_sm(sm: int) -> bool: + """Whether the TRTLLM attention op runs the trtllm-gen kernels on ``sm``.""" + return not (sm < 100 or sm in [120, 121]) + + +@functools.cache +def generate_spec_decoding_position_offsets(max_num_requests: int, draft_len: int) -> torch.Tensor: + width = draft_len + 1 + row = torch.arange(width, dtype=torch.int, device="cuda") + return row.unsqueeze(0).expand(max_num_requests, -1).contiguous() + + +@functools.cache +def generate_spec_decoding_packed_mask(max_num_requests: int, draft_len: int) -> torch.Tensor: + width = draft_len + 1 + num_blocks = math.ceil(width / 32) + mask = torch.zeros([max_num_requests, width, num_blocks], dtype=torch.int, device="cuda") + remaining = width + for blk in range(num_blocks): + if remaining <= 0: + break + n = min(32, remaining) + vals = (torch.pow(2, torch.arange(n) + 1) - 1).int() + mask[:, blk * 32 : blk * 32 + n, blk] = vals + remaining -= 32 + return mask + + +@dataclass(frozen=True) +class SpecDecodingConfig: + """Speculative-decoding settings fixed for the lifetime of a metadata. + + A metadata takes them at creation (``spec_decoding``) and sizes its + draft-token buffers from them; every CUDA graph copy is created with the + same settings. + """ + + #: Whether the attention runs in multi-query (spec-dec) mode; see + #: ``SpeculativeDecodingMode.attention_need_spec_dec_mode``. + attention_needs_spec_dec_mode: bool = False + #: Whether the draft tokens form a tree, and whether it changes per step. + is_spec_dec_tree: bool = False + is_spec_dec_dynamic_tree: bool = False + #: The number of draft layers and of draft tokens (tree nodes but the root). + max_draft_len: int = 0 + max_total_draft_tokens: int = 0 + #: Per-request row width of the tree buffers (the tree manager's internal + #: dimension); 0 for linear drafts. + tree_buffer_dim: int = 0 + + +def set_use_spec_decoding(metadata, use: bool) -> None: + """Toggle spec-dec mode for this step on metadata whose backend has one.""" + spec = metadata.find_ext(SpecDecodingMetadata) + if spec is not None: + spec.use_spec_decoding = use + + +class SpecDecodingMetadata(MetadataExtension): + """Spec-dec mode flags and the draft-token buffers the attention op reads. + + The buffers stay None until ``update`` enables spec-dec mode; CUDA graph + copies share them with the metadata they were copied from. + """ + + # A draft binds the target's spec-dec settings and step flags. Its loop + # rewrites the masks, position offsets and generation lengths in place, + # so it keeps buffers of its own for them, copied from the target's on + # device (not the buffers the target shares with its CUDA graph copies). + adopts_target_state = True + # Every draft takes it at its draft-loop start, also one that derives the + # rest: it depends on neither the KV cache nor other extensions (the + # engine sets it on the target with update_spec_dec_param). + adopts_target_state_always = True + draft_bound = frozenset( + { + "spec_decoding", + "max_num_heads_per_kv", + "is_spec_decoding_enabled", + "use_spec_decoding", + "is_spec_dec_tree", + "is_spec_dec_dynamic_tree", + "force_prepare_spec_dec_tree_mask", + "max_total_draft_tokens", + } + ) + draft_written = frozenset( + { + "spec_decoding_query_len", + "spec_decoding_position_offsets", + "spec_decoding_packed_mask", + "spec_decoding_generation_lengths", + "spec_decoding_bl_tree_mask_offset", + "spec_decoding_bl_tree_mask", + "spec_bl_tree_first_sparse_mask_offset_kv", + } + ) + + fields = frozenset( + { + "spec_decoding", + "max_num_heads_per_kv", + "spec_bl_tree_first_sparse_mask_offset_kv", + "spec_decoding_bl_tree_mask", + "spec_decoding_bl_tree_mask_offset", + "force_prepare_spec_dec_tree_mask", + "spec_decoding_generation_lengths", + "is_spec_dec_dynamic_tree", + "is_spec_dec_tree", + "is_spec_decoding_enabled", + "max_total_draft_tokens", + "spec_decoding_packed_mask", + "spec_decoding_position_offsets", + "position_offsets_for_cpp", + "spec_decoding_query_len", + "use_spec_decoding", + } + ) + methods = frozenset( + { + "fill_generation_lengths", + "spec_decoding_param_prepare_for_blackwell", + "update", + "set_spec_decoding_query_len", + "update_blackwell_first_sparse_mask_offset", + } + ) + + init_fields = { + # Upper bound on query heads per KV head over the target model's layers + # (unsharded); sizes the spec-dec tree mask. + "max_num_heads_per_kv": 1, + # The speculative-decoding settings this metadata was created with. + "spec_decoding": SpecDecodingConfig(), + } + # The dynamic tree writes these each step on the metadata the engine + # prepares; its CUDA graph copies read them through the shared tensors. + shared = frozenset( + { + "spec_decoding_position_offsets", + "spec_decoding_packed_mask", + "spec_decoding_generation_lengths", + "spec_decoding_bl_tree_mask", + "spec_decoding_bl_tree_mask_offset", + "spec_bl_tree_first_sparse_mask_offset_kv", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Spec-dec mode (multi-query mode) of the TRTLLM XQA kernels, supported + # for the entire runtime. + self.is_spec_decoding_enabled = False + # Whether the attention layer runs in spec-dec mode at this step/layer. + self.use_spec_decoding = False + # Whether the spec-dec tree is a tree or a chain (linear tree). + self.is_spec_dec_tree = False + # A tree that never changes needs its mask computed only once. + self.is_spec_dec_dynamic_tree = False + self.force_prepare_spec_dec_tree_mask = False + self.max_total_draft_tokens: Optional[int] = None + self.spec_decoding_position_offsets: Optional[torch.Tensor] = self.shared_or( + "spec_decoding_position_offsets", lambda: None + ) + # Current query width of the compact prefix in a 1-D dynamic-tree + # offsets buffer. + self.spec_decoding_query_len = 0 + self.spec_decoding_packed_mask: Optional[torch.Tensor] = self.shared_or( + "spec_decoding_packed_mask", lambda: None + ) + self.spec_decoding_generation_lengths: Optional[torch.Tensor] = self.shared_or( + "spec_decoding_generation_lengths", lambda: None + ) + self.spec_decoding_bl_tree_mask_offset: Optional[torch.Tensor] = self.shared_or( + "spec_decoding_bl_tree_mask_offset", lambda: None + ) + self.spec_decoding_bl_tree_mask: Optional[torch.Tensor] = self.shared_or( + "spec_decoding_bl_tree_mask", lambda: None + ) + self.spec_bl_tree_first_sparse_mask_offset_kv: Optional[torch.Tensor] = self.shared_or( + "spec_bl_tree_first_sparse_mask_offset_kv", lambda: None + ) + config = self.spec_decoding + md = self.metadata + trtllm_gen = is_trtllm_gen_sm(get_sm_version()) + # Blackwell trtllm-gen spec-dec is enabled only for dynamic-tree masks. + self.is_spec_decoding_enabled = config.attention_needs_spec_dec_mode and ( + not trtllm_gen or config.is_spec_dec_dynamic_tree + ) + self.is_spec_dec_tree = config.is_spec_dec_tree + self.is_spec_dec_dynamic_tree = config.is_spec_dec_dynamic_tree + # A hybrid model's first executed attention layer can have a nonzero + # cache-local index because recurrent layers precede it. Do not rely + # on the C++ ``layer_idx == 0`` fallback to rebuild the target mask: + # the dynamic draft loop clears that mask before the next target step. + self.force_prepare_spec_dec_tree_mask = config.is_spec_dec_dynamic_tree + # Forward the tree length to FMHA kernel selection. + if config.max_draft_len or config.max_total_draft_tokens: + self.max_total_draft_tokens = config.max_total_draft_tokens + if not self.is_spec_decoding_enabled: + return + + # These buffers are accessed more like removing input padding, + # rather than using max_total_draft_tokens + 1 as the offset between different requests. + if config.is_spec_dec_tree and self.spec_decoding_position_offsets is None: + # Dynamic tree: the tree manager's internal dimension may be + # larger than max_total_draft_tokens+1 to accommodate + # K*max_draft_len. 1D layout for flexible view() in the drafting loop. + self.spec_decoding_position_offsets = torch.empty( + (md.max_num_requests * config.tree_buffer_dim,), + dtype=torch.int, + device="cuda", + ) + if config.is_spec_dec_tree and self.spec_decoding_packed_mask is None: + buf_dim = config.tree_buffer_dim + # Zero-init: dynamic-tree dst has inner dim + # ceil(buf_dim/32) but only ceil((max_total_draft_tokens+1)/32) + # is written each step. Unwritten cols would otherwise feed the + # C++ XQA kernel as live mask bits. + self.spec_decoding_packed_mask = torch.zeros( + [md.max_num_requests, buf_dim, math.ceil(buf_dim / 32)], + dtype=torch.int, + device="cuda", + ) + if self.spec_decoding_generation_lengths is None: + self.spec_decoding_generation_lengths = torch.empty( + [md.max_num_requests], + dtype=torch.int, + device="cuda", + ) + if trtllm_gen: + self.spec_decoding_param_prepare_for_blackwell() + + @property + def position_offsets_for_cpp(self) -> Optional[torch.Tensor]: + """``spec_decoding_position_offsets`` as the [requests, query width] view the attention op reads. + + A 1-D (dynamic-tree) buffer is viewed at its current compact query + width on Hopper, which can be smaller than the buffer's capacity, and + at its full row width on trtllm-gen. + """ + offsets = self.spec_decoding_position_offsets + if offsets is None or offsets.dim() != 1: + return offsets + max_num_requests = self.metadata.max_num_requests + if is_trtllm_gen_sm(get_sm_version()): + return offsets.view(max_num_requests, -1) + query_len = self.spec_decoding_query_len + if query_len <= 0: + raise ValueError("1-D speculative position offsets require a positive query length.") + return offsets[: max_num_requests * query_len].view(max_num_requests, query_len) + + def set_spec_decoding_query_len(self, query_len: int) -> None: + """Set the query width of the compact prefix of the 1-D offsets buffer. + + A dynamic-tree draft loop narrows it between its draft forwards. + """ + self.spec_decoding_query_len = query_len + + def spec_decoding_param_prepare_for_blackwell(self) -> None: + """Prepare the Blackwell parameters of the generation-phase spec-dec kernels. + + Uses persistent buffers (only allocated if None) for CUDA graph + compatibility. + """ + md = self.metadata + if self.spec_decoding_bl_tree_mask_offset is None: + self.spec_decoding_bl_tree_mask_offset = torch.zeros( + [md.max_num_requests], + dtype=torch.int64, + device="cuda", + ) + + if self.spec_bl_tree_first_sparse_mask_offset_kv is None: + self.spec_bl_tree_first_sparse_mask_offset_kv = torch.zeros( + [md.max_num_requests], + dtype=torch.int32, + device="cuda", + ) + + if self.spec_decoding_bl_tree_mask is None: + seqLenQ = ( + self.spec_decoding_packed_mask.shape[1] + if self.spec_decoding_packed_mask is not None + else 1 + ) + mask_size = torch.ops.trtllm.spec_dec_tree_custom_mask_capacity( + md.max_num_requests, seqLenQ, self.max_num_heads_per_kv + ) + self.spec_decoding_bl_tree_mask = torch.zeros( + mask_size, + dtype=torch.uint32, + device="cuda", + ) + + def update_blackwell_first_sparse_mask_offset(self) -> None: + """Fill gen slots [0:ng) with (kv_lens_cuda - seq_lens); in-place for CUDA graph.""" + md = self.metadata + nc = md.num_contexts + n = md.num_seqs + ng = n - nc + if ng > 0: + gen_offset = (md.kv_lens_cuda[nc:n] - md.seq_lens_cuda[nc:n]).to(torch.int32) + self.spec_bl_tree_first_sparse_mask_offset_kv[:ng].copy_(gen_offset) + + def fill_generation_lengths(self, runtime_draft_len: int) -> None: + self.spec_decoding_generation_lengths[: self.metadata.max_num_requests].fill_( + runtime_draft_len + 1 + ) + + def update( + self, + batch_size, + spec_metadata: Optional["SpecMetadata"] = None, + spec_tree_manager: Optional["SpecTreeManager"] = None, + num_contexts: int = 0, + ) -> None: + """Write this step's draft-token layout into the spec-dec buffers. + + Args: + batch_size: int, the number of requests in the batch. + spec_metadata: Optional['SpecMetadata'] = None, the metadata of the spec-dec. + spec_tree_manager: Optional['SpecTreeManager'] = None, the spec_tree_manager for draft token tree. + num_contexts: int = 0, the number of context (prefill) requests in the + batch. The slot-storage layout is ``[ctx | gen]``; dynamic-tree + metadata only describes the gen rows, so we must source from + ``[num_contexts:batch_size]`` rather than ``[:batch_size]``. + """ + md = self.metadata + trtllm_gen = is_trtllm_gen_sm(get_sm_version()) + # use_spec_decoding is default to true by default, change in runtime by layers / requests + self.use_spec_decoding = self.is_spec_decoding_enabled + if not self.is_spec_decoding_enabled: + return + max_draft_len = self.spec_decoding.max_draft_len + max_total_draft_tokens = self.spec_decoding.max_total_draft_tokens + + # Case 1: dynamic tree — copy per-request params from spec_tree_manager. + if self.is_spec_dec_dynamic_tree: + assert spec_tree_manager is not None, "spec_tree_manager is required for dynamic tree" + n_dt = spec_tree_manager.max_total_draft_tokens + 1 + buf_dim = spec_tree_manager._internal_buf_dim + + assert buf_dim >= n_dt, ( + f"Dynamic-tree copy requires buf_dim >= n_dt; got buf_dim={buf_dim}, n_dt={n_dt}." + ) + + slot_storage = spec_tree_manager.slot_storage + + num_gens = batch_size - num_contexts + if num_gens > 0: + slot_ids = slot_storage.all_ids_buf[num_contexts:batch_size] + + pos_src = torch.index_select( + slot_storage.spec_decoding_position_offsets, 0, slot_ids + )[:, :n_dt] + pos_dst = self.spec_decoding_position_offsets[: num_gens * n_dt].view( + num_gens, n_dt + ) + pos_dst.copy_(pos_src, non_blocking=True) + + actual_mask_width = math.ceil(n_dt / 32) + mask_src = torch.index_select(slot_storage.spec_decoding_packed_mask, 0, slot_ids)[ + :, :, :actual_mask_width + ] + if trtllm_gen: + # Blackwell reads the padded 3D mask layout. + self.spec_decoding_packed_mask[:num_gens, :n_dt, :actual_mask_width].copy_( + mask_src, non_blocking=True + ) + else: + # Hopper XQA reads a compact flat prefix. + total = num_gens * n_dt * actual_mask_width + self.spec_decoding_packed_mask.view(-1)[:total].copy_( + mask_src.reshape(-1), non_blocking=True + ) + + self.spec_decoding_generation_lengths[:batch_size].fill_(n_dt) + self.spec_decoding_query_len = n_dt + + # Case 2: linear tree + else: + # Currently dynamic draft length is only supported for linear tree + # Dynamic draft length needs position offsets and packed mask to be shaped for each runtime draft length. + # So we create cache for position offsets and packed mask for each draft length to avoid reallocation. + assert max_draft_len == max_total_draft_tokens, ( + "max_draft_len should be equal to max_total_draft_tokens for linear tree" + ) + # For algos other than PARD, this equals runtime_draft_len (K); for PARD it's 2K-1. + runtime_draft_token_buffer_width = ( + spec_metadata.runtime_tokens_per_gen_step - 1 + if spec_metadata is not None + else max_draft_len + ) + self.fill_generation_lengths(runtime_draft_len=runtime_draft_token_buffer_width) + self.spec_decoding_position_offsets = generate_spec_decoding_position_offsets( + md.max_num_requests, runtime_draft_token_buffer_width + ) + self.spec_decoding_packed_mask = generate_spec_decoding_packed_mask( + md.max_num_requests, runtime_draft_token_buffer_width + ) + self.spec_decoding_query_len = runtime_draft_token_buffer_width + 1 diff --git a/tensorrt_llm/_torch/attention/backends/flashinfer.py b/tensorrt_llm/_torch/attention/backends/flashinfer.py index 0c7b4433426d..4c77a79e46cf 100644 --- a/tensorrt_llm/_torch/attention/backends/flashinfer.py +++ b/tensorrt_llm/_torch/attention/backends/flashinfer.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import functools import math import os @@ -20,18 +19,21 @@ import weakref from dataclasses import dataclass, field from itertools import chain -from typing import Any, Dict, NewType, Optional, TypeAlias, cast +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, NewType, Optional, Type, + TypeAlias, cast) if sys.version_info[:2] >= (3, 12): from typing import override else: from typing_extensions import override +if TYPE_CHECKING: + from ...pyexecutor.kv_cache.mixed_precision_staging import MixedPrecisionKVStaging + import flashinfer import numpy as np import torch from flashinfer.jit.core import check_cuda_arch -from typing_extensions import Self from tensorrt_llm._utils import nvtx_range from tensorrt_llm.functional import AttentionMaskType @@ -41,6 +43,7 @@ from ...metadata import KVCacheParams from ...utils import get_global_attrs, get_model_extra_attrs, torch_multi_arange +from .core import AttentionCoreMetadata from .interface import (AttentionBackend, AttentionForwardArgs, AttentionInputType, AttentionMetadata, CustomAttentionMask, MLAParams, PredefinedAttentionMask, @@ -175,7 +178,7 @@ def _append_paged_kv_cache( # Perf knob for the FlashInfer decode wrapper's kernel choice; see -# ``FlashInferAttentionMetadata._use_tensor_cores``. "auto" (default) keeps +# ``FlashInferCoreMetadata._use_tensor_cores``. "auto" (default) keeps # FlashInfer's GQA-ratio heuristic, "0" forces the split-K decode kernel, "1" # forces the tensor-core (paged-prefill) one. It never applies where only the # tensor-core kernel exists. @@ -305,1724 +308,1740 @@ class FlashInferWrappers: _PAGE_TABLE_CHECK_REPORTS = 20 -@dataclass(kw_only=True) -class FlashInferAttentionMetadata(AttentionMetadata): - # Report budget for the page-table check. A class attribute, because it - # bounds a run's log rather than a batch's, and deliberately unannotated so - # that @dataclass does not turn it into a field. - _page_table_check_budget = _PAGE_TABLE_CHECK_REPORTS - - workspace_buffer: Optional[torch.Tensor] = None - - paged_kv_indptr_decode: torch.Tensor = field(init=False) - paged_kv_indptr_prefill: torch.Tensor = field(init=False) - _paged_kv_indices: torch.Tensor = field(init=False, repr=False) - _paged_kv_last_page_len: torch.Tensor = field(init=False) - _qo_indptr: torch.Tensor = field(init=False) - _kv_indptr: torch.Tensor = field(init=False) - _ragged_qo_indptr_buf: torch.Tensor = field(init=False) - _ragged_kv_indptr_buf: torch.Tensor = field(init=False) - _cached_token_lens: torch.Tensor = field(init=False) - # Attention-visible KV lengths. These exclude pages reserved for future speculative tokens when - # the structural page table exposes extra capacity. - _logical_kv_lens: torch.Tensor = field(init=False, repr=False) - _plan_params_to_wrappers: Dict[PlanParams, - FlashInferWrappers] = field(init=False) - - # MLA wrappers and stable buffers. - # Cached plan params + is-planned flag let prepare() refresh the plan - # outside stream capture (flashinfer plan() does device->host syncs). - _ragged_prefill_wrapper: Optional[ - flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper] = field( - init=False, default=None) - _mla_decode_wrapper: Optional[object] = field(init=False, default=None) - _mla_context_wrapper: Optional[object] = field(init=False, default=None) - _mla_ragged_plan_params: Optional[RaggedPlanParams] = field(init=False, - default=None) - _mla_context_plan_params: Optional[MLAPlanParams] = field(init=False, - default=None) - _mla_decode_plan_params: Optional[MLAPlanParams] = field(init=False, - default=None) - num_ctx_cached_tokens: int = field(init=False, default=0) - _mla_ragged_planned: bool = field(init=False, default=False) - _mla_context_planned: bool = field(init=False, default=False) - _mla_decode_planned: bool = field(init=False, default=False) - _mla_qo_indptr_buf: Optional[torch.Tensor] = field(init=False, default=None) - _mla_kv_len_arr_buf: Optional[torch.Tensor] = field(init=False, - default=None) - - _multi_item_params: Optional[FlashInferMultiItemParams] = field( - init=False, default=None) - _draft_metadata: Optional["FlashInferAttentionMetadata"] = field( - init=False, default=None, repr=False) - _draft_kv_runtime_lens: torch.Tensor = field(init=False, repr=False) - _is_shared_kv_draft_view: bool = field(init=False, - default=False, - repr=False) - _is_separate_kv_draft_view: bool = field(init=False, - default=False, - repr=False) - _uses_full_draft_page_table: bool = field(init=False, - default=False, - repr=False) - # Whether target generation rows expose reserved speculative pages while keeping their - # attention-visible lengths in `_logical_kv_lens`. - _uses_full_generation_page_table: bool = field(init=False, - default=False, - repr=False) - - def needs_plan(self, plan_params: PlanParams) -> bool: - if plan_params not in self._plan_params_to_wrappers: - return True - - wrappers = self._plan_params_to_wrappers[plan_params] - return not wrappers.is_planned - - def get_prefill_wrapper( - self, plan_params: PlanParams - ) -> flashinfer.BatchPrefillWithPagedKVCacheWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[plan_params].prefill_wrapper - assert result is not None, "Prefill wrapper was not created in plan()" - return result +class FlashInferCoreMetadata(AttentionCoreMetadata): + """FlashInfer attention state: paged-KV tables, planner wrappers and VSWA buffers. - def get_decode_wrapper( - self, plan_params: PlanParams - ) -> flashinfer.BatchDecodeWithPagedKVCacheWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[plan_params].decode_wrapper - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - if result._backend != "trtllm-gen": - raise ValueError( - "FlashInfer draft metadata views require the trtllm-gen " - "decode backend.") - self._publish_decode_wrapper_kv_lens(result) - return result + ``allocate`` binds the graph-stable buffers on creation and on every + metadata copy; ``derive`` builds the step's paged-KV tables and plans. + """ - def get_ragged_prefill_wrapper( - self, plan_params: PlanParams) -> _RaggedPrefillWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[ - plan_params].ragged_prefill_wrapper - assert result is not None, "Ragged prefill wrapper was not created in plan()" - return result + # Copies run on the same workspace and the same per-pool page buffers. + shared = frozenset({"workspace_buffer", "_vswa_pool_bufs"}) + fields = frozenset({ + "batch_indices", + "cached_token_lens", + "kv_indptr", + "kv_lens_cuda", + "max_seq_len", + "num_blocks", + "num_context_blocks", + "num_ctx_cached_tokens", + "num_generation_blocks", + "page_size", + "paged_kv_indices", + "paged_kv_indptr", + "paged_kv_indptr_decode", + "paged_kv_indptr_prefill", + "paged_kv_last_page_len", + "positions", + "qo_indptr", + "workspace_buffer", + }) + methods = frozenset({ + "apply_spec_decode_kv_lens_offsets", + "get_decode_wrapper", + "get_draft_metadata", + "get_paged_kv_indices_for_layer", + "get_prefill_wrapper", + "get_ragged_prefill_wrapper", + "needs_plan", + "plan", + "plan_mla_context", + "plan_mla_decode", + "plan_ragged", + "swap_paged_kv_indices_for_layer", + "update_shared_kv_draft_lengths", + }) + init_fields = { + "workspace_buffer": None, + } + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # MLA wrappers and stable buffers. + # Cached plan params + is-planned flag let prepare() refresh the plan + # outside stream capture (flashinfer plan() does device->host syncs). + self._ragged_prefill_wrapper = None + self._mla_decode_wrapper = None + self._mla_context_wrapper = None + self._mla_ragged_plan_params = None + self._mla_context_plan_params = None + self._mla_decode_plan_params = None + self.num_ctx_cached_tokens = 0 + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False + self._mla_qo_indptr_buf = None + # Graph-stable page-index buffer of each VSWA pool, keyed by pool ID. CUDA + # graph copies share their source's buffers; a separate-KV draft view + # allocates its own. + self._vswa_pool_bufs = self.shared_or("_vswa_pool_bufs", lambda: dict()) + self._mla_kv_len_arr_buf = None + self._multi_item_params = None + self._draft_metadata = None + self._is_shared_kv_draft_view = False + self._is_separate_kv_draft_view = False + self._uses_full_draft_page_table = False + # Whether target generation rows expose reserved speculative pages while keeping their + # attention-visible lengths in `_logical_kv_lens`. + self._uses_full_generation_page_table = False + self.workspace_buffer = self.shared_or("workspace_buffer", + lambda: self.workspace_buffer) + buffers = metadata.cuda_graph_buffers + md = self.metadata + capture_graph = self.is_cuda_graph - def plan_ragged( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - plan_params: RaggedPlanParams, - ) -> flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper: - """Plan MLA ragged prefill with expanded K, V (not paged).""" - if self._ragged_prefill_wrapper is None: - self._ragged_prefill_wrapper = flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper( - self.workspace_buffer, - "NHD", + if self.workspace_buffer is None: + # Note: even though flashinfer only recommends 128 MB, we have to push it + # a bit higher to cover all possible CUDA graph cases. If it's too small, + # warmup will crash. + # FlashInfer's split-KV decode plan sizes its temp buffers proportionally to + # SM count; 320 MB was tuned for a 148-SM baseline, so scale up (never down) + # on GPUs with more SMs. + baseline_sms = 148 + baseline_bytes = 320 * 1024 * 1024 + num_sms = torch.cuda.get_device_properties( + torch.cuda.current_device()).multi_processor_count + workspace_bytes = max(baseline_bytes, + baseline_bytes * num_sms // baseline_sms) + self.workspace_buffer = md.get_empty( + buffers, + (workspace_bytes, ), + dtype=torch.uint8, + cache_name="workspace_buffer", + capture_graph=capture_graph, ) - if self._mla_ragged_plan_params != plan_params: - self._mla_ragged_planned = False - self._mla_ragged_plan_params = plan_params - - if self._mla_ragged_planned: - return self._ragged_prefill_wrapper - - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA ragged prefill while the stream " - "is capturing. Make sure prepare() has run at least one " - "warmup forward pass before capture.") - # Split append_paged_mla_kv_cache from plan() when this wrapper needs a - # new plan. Reusing a cached plan avoids this sync on later layers. - torch.cuda.current_stream().synchronize() + self.paged_kv_indptr_decode = torch.empty((self.max_num_requests + 1, ), + device='cuda', + dtype=torch.int) + self.paged_kv_indptr_prefill = torch.empty( + (self.max_num_requests + 1, ), device='cuda', dtype=torch.int) + self._paged_kv_last_page_len = torch.empty((self.max_num_requests, ), + device='cuda', + dtype=torch.int) - self._do_plan_ragged(qo_indptr, kv_indptr, plan_params) - self._mla_ragged_planned = True + self._qo_indptr = torch.zeros(self.max_num_requests + 1, + device='cuda', + dtype=torch.int) - return self._ragged_prefill_wrapper + self._kv_indptr = torch.zeros( + self.max_num_requests + 1, device='cuda', + dtype=torch.int) if self.is_cross else self._qo_indptr - def _do_plan_ragged( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - plan_params: RaggedPlanParams, - ) -> None: - assert self._ragged_prefill_wrapper is not None - self._ragged_prefill_wrapper.plan( - qo_indptr, - kv_indptr, - plan_params.num_heads, - plan_params.num_kv_heads, - plan_params.head_dim, - head_dim_vo=plan_params.head_dim_vo, - use_fp16_qk_reduction=False, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, + self._ragged_qo_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_ragged_qo_indptr_buf", + capture_graph=capture_graph, + ) + self._ragged_kv_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_ragged_kv_indptr_buf", + capture_graph=capture_graph, ) - def plan_mla_decode( - self, - plan_params: MLAPlanParams, - ) -> object: - """Plan MLA decode using BatchMLAPagedAttentionWrapper. - - Caches the wrapper and plan params; the actual plan() call is driven - by prepare() so it runs outside of CUDA graph capture. - """ - if self._mla_decode_wrapper is None: - self._mla_decode_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( - self.workspace_buffer, - use_cuda_graph=self.is_cuda_graph, - qo_indptr=self._mla_qo_indptr_buf, - kv_indptr=self.paged_kv_indptr_decode, - kv_indices=self._paged_kv_indices, - kv_len_arr=self._mla_kv_len_arr_buf, - backend="auto", - ) - - if self._mla_decode_plan_params != plan_params: - self._mla_decode_planned = False + self._cached_token_lens = torch.empty((self.max_num_requests, ), + dtype=torch.int, + device='cuda') + self._logical_kv_lens = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int, + cache_name="_logical_kv_lens", + capture_graph=capture_graph, + ) + self._draft_kv_runtime_lens = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int, + cache_name="_draft_kv_runtime_lens", + capture_graph=capture_graph, + ) + self._batch_indices = torch.empty((self.max_num_tokens, ), + dtype=torch.int, + device='cuda') + self._positions = torch.empty((self.max_num_tokens, ), + dtype=torch.int, + device='cuda') + self._plan_params_to_wrappers = {} - # Cache params so prepare() can re-plan on subsequent forward passes. - self._mla_decode_plan_params = plan_params + # Host-side state for sync-free trtllm-gen decode plans, retained by + # prepare(): a host mirror of _paged_kv_indices (kept in lockstep + # with the device buffer, with per-pool copies for VSWA swaps) and a + # host copy of the decode indptr. The per-plan_params persistent + # block-table buffers live on FlashInferWrappers. + self._host_pool_indices: Dict[int, torch.Tensor] = {} + self._host_paged_kv_indices: Optional[torch.Tensor] = None + self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None + self._uses_full_generation_page_table = False + self._max_num_blocks_per_seq = 0 - if self._mla_decode_planned: - return self._mla_decode_wrapper + # VSWA (Variable Sliding Window Attention): models with per-layer + # max_attention_window create separate V2 pool groups with independent + # page numbering. We need per-pool paged_kv_indices so each layer can + # use the indices that match its pool's buffer. + self._vswa_layer_to_pool: Optional[Dict[int, int]] = None + self._vswa_pool_indices_cache: Optional[Dict[int, torch.Tensor]] = None + self._primary_kv_layer_idx: Optional[int] = None + self._primary_kv_pool_id = 0 - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA decode while the stream is " - "capturing. Make sure prepare() has run at least one warmup " - "forward pass before capture.") + if self.kv_cache_manager is not None: + blocks_in_primary_pool = self.kv_cache_manager.blocks_in_primary_pool + attention_layer_indices = _get_attention_layer_indices( + self.kv_cache_manager) - # Split append_paged_mla_kv_cache from plan() on a cache miss. prepare() - # calls _do_plan_mla_decode() directly and does not need this sync. - torch.cuda.current_stream().synchronize() - self._do_plan_mla_decode(plan_params) - self._mla_decode_planned = True - return self._mla_decode_wrapper + # Maximum block count across attention pools: sizes the shared + # page-index buffer and the VSWA pool buffers below. + max_num_blocks = blocks_in_primary_pool + for layer_idx in attention_layer_indices: + layer_buffer = self.kv_cache_manager.get_buffers(layer_idx) + if layer_buffer is not None: + max_num_blocks = max(max_num_blocks, layer_buffer.shape[0]) + if self._has_mixed_precision_staging(): + # The staging page space spans every sequence slot. + max_num_blocks = max( + max_num_blocks, + (self.max_num_sequences or self.max_num_requests) * + self.kv_cache_manager.max_blocks_per_seq) + self._paged_kv_indices = md.get_empty( + buffers, + (max_num_blocks, ), + dtype=torch.int, + cache_name="_paged_kv_indices", + capture_graph=capture_graph, + ) + self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq - def plan_mla_context( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - kv_last_page_len: torch.Tensor, - plan_params: MLAPlanParams, - ) -> object: - """Plan MLA context with cached KV using BatchMLAPagedAttentionWrapper.""" - if self._mla_context_wrapper is None: - self._mla_context_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( - self.workspace_buffer, - use_cuda_graph=False, - backend="auto", - ) - - if self._mla_context_plan_params != plan_params: - self._mla_context_planned = False - self._mla_context_plan_params = plan_params - - if self._mla_context_planned: - return self._mla_context_wrapper - - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA context while the stream is " - "capturing. Make sure prepare() has run at least one warmup " - "forward pass before capture.") - - # Split append_paged_mla_kv_cache from plan() when this wrapper needs a - # new plan. Reusing a cached plan avoids this sync on later layers. - torch.cuda.current_stream().synchronize() - - self._do_plan_mla_context( - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_len, - plan_params, + # Layers may share one page-index list only when they are in the + # same pool, page-index scale, and KV factor: VSWA splits pools, + # and per-layer geometry (e.g. Gemma4 sliding/global head_dim) + # splits scales even when all windows collapse to max_seq_len and + # is_vswa is False. Mixed K-only/KV layers can use different view + # indices even with the same physical scale. Guarded on V2-specific + # attributes (V1 managers lack the per-pool infrastructure). + mgr = self.kv_cache_manager + get_scale = getattr(mgr, 'get_layer_page_index_scale', None) + get_kv_factor = getattr(mgr, 'get_layer_kv_factor', None) + layer_space: Dict[int, int] = {} + if hasattr(mgr, 'layer_to_pool_mapping_dict') and get_scale: + space_ids = {} + for layer_idx in attention_layer_indices: + layer_offset = mgr.layer_offsets[layer_idx] + key = (mgr.layer_to_pool_mapping_dict[layer_offset], + get_scale(layer_idx), + get_kv_factor(layer_idx) if get_kv_factor else None) + if (self._has_mixed_precision_staging() and layer_offset + in mgr.mixed_precision_high_layers): + key = ("mixed_precision_staging", ) + space_ids.setdefault(key, len(space_ids)) + layer_space[layer_idx] = space_ids[key] + if layer_space: + self._primary_kv_layer_idx = next(iter(layer_space)) + self._primary_kv_pool_id = layer_space[ + self._primary_kv_layer_idx] + if layer_space and (getattr(mgr, 'is_vswa', False) + or len(set(layer_space.values())) > 1): + self._vswa_layer_to_pool = {} + self._vswa_pool_to_rep_layer: Dict[int, int] = {} + for layer_idx, pool_id in layer_space.items(): + self._vswa_layer_to_pool[layer_idx] = pool_id + if pool_id not in self._vswa_pool_to_rep_layer: + self._vswa_pool_to_rep_layer[pool_id] = layer_idx + # Pre-allocate VSWA pool cache buffers. These must be + # stable (never reallocated) so that CUDA-graph-recorded + # copies reference valid addresses across replays. + # max_num_blocks (computed above) covers ALL pools so that + # secondary pool buffers are large enough. + for pool_id in set(self._vswa_layer_to_pool.values()): + if self._vswa_pool_bufs.get(pool_id) is None: + self._vswa_pool_bufs[pool_id] = torch.empty( + max_num_blocks, dtype=torch.int, device='cuda') + # Stable buffers for FlashInfer MLA decode; required for CUDA graphs. + self._mla_qo_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_mla_qo_indptr_buf", + capture_graph=capture_graph, ) - self._mla_context_planned = True + self._mla_kv_len_arr_buf = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int32, + cache_name="_mla_kv_len_arr_buf", + capture_graph=capture_graph, + ) + # Rebind the wrapper to the freshly allocated buffers. + self._ragged_prefill_wrapper = None + self._mla_decode_wrapper = None + self._mla_context_wrapper = None + self._mla_ragged_plan_params = None + self._mla_context_plan_params = None + self._mla_decode_plan_params = None + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False + self._multi_item_params = None - return self._mla_context_wrapper + def derive(self) -> None: + md = self.metadata - def _do_plan_mla_context( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - kv_last_page_len: torch.Tensor, - plan_params: MLAPlanParams, - ) -> None: - assert self._mla_context_wrapper is not None + def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: + """Host int32 staging tensor for async H2D copies; pinned when + beneficial. - num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] - kv_len_arr = (num_pages_per_seq - - 1) * plan_params.page_size + kv_last_page_len + torch.tensor(..., pin_memory=True) rejects numpy sources, hence + from_numpy + pin. + """ + return maybe_pin_memory( + torch.from_numpy(arr.astype(np.int32, copy=False))) - self._mla_context_wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - kv_len_arr, - plan_params.num_heads, - plan_params.kv_lora_rank, - plan_params.qk_rope_head_dim, - plan_params.page_size, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, - ) + extra_attrs = get_model_extra_attrs() + if extra_attrs is None: + get_global_attrs().attention_metadata = weakref.ref(md) + # start and end indices of each sequence in the ragged query + assert self.seq_lens_cuda is not None + torch.cumsum(self.seq_lens_cuda, + dim=0, + dtype=torch.int32, + out=self._qo_indptr[1:self.seq_lens_cuda.size(0) + 1]) - def _do_plan_mla_decode(self, plan_params: MLAPlanParams) -> None: - """Compute MLA decode plan inputs and call wrapper.plan(). + if self.multi_item_part_lens is not None: + self._multi_item_params = self._process_multi_item_part_lens( + self.multi_item_part_lens, device=self.seq_lens_cuda.device) + else: + self._multi_item_params = None - Must run outside of CUDA graph capture. - """ - num_gen = self.num_generations - kv_indptr = self.paged_kv_indptr_decode[:num_gen + 1] - kv_indices = self._paged_kv_indices[self.num_context_blocks:self. - num_context_blocks + - self.num_generation_blocks] - kv_last_page = self._paged_kv_last_page_len[self.num_contexts:self. - num_contexts + num_gen] + if self.kv_cache_manager is None: + assert self.request_ids is not None + assert self.num_generations == 0, ( + "FlashInfer without a KV cache manager only supports context-only " + "batches (num_generations == 0) in TRT-LLM.") + if self.is_cross: + raise NotImplementedError( + "FlashInfer without a KV cache manager is not tested for cross attention." + ) + self.kv_cache_params = KVCacheParams(use_cache=False) + n = self.num_seqs + self._cached_token_lens[:n].zero_() + self.num_ctx_cached_tokens = 0 + self._clean_cached_plans(defer_plan=False) + return - # _qo_indptr is ordered [context_seqs..., generation_seqs...]; rebase - # the generation slice to 0. - num_ctx = self.num_contexts - qo_indptr = self._qo_indptr[num_ctx:num_ctx + num_gen + - 1] - self._qo_indptr[num_ctx] + if self._multi_item_params is not None: + raise ValueError( + "multi_item_part_lens with KV cache is not supported") - if self._uses_full_generation_page_table: - kv_len_arr = self._logical_kv_lens[num_ctx:num_ctx + num_gen] + # number of tokens in the kv cache for each sequence in the batch + num_cached_tokens_per_seq = self.kv_cache_params.num_cached_tokens_per_seq + cached_token_lens = torch.tensor(num_cached_tokens_per_seq, + dtype=torch.int, + pin_memory=prefer_pinned()) + self._cached_token_lens[:cached_token_lens.size(0)].copy_( + cached_token_lens, non_blocking=True) + if self.num_contexts > 0: + self.num_ctx_cached_tokens = sum( + num_cached_tokens_per_seq[:self.num_contexts]) else: - num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] - kv_len_arr = (num_pages_per_seq - - 1) * plan_params.page_size + kv_last_page + self.num_ctx_cached_tokens = 0 - self._mla_decode_wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - kv_len_arr, - plan_params.num_heads, - plan_params.kv_lora_rank, - plan_params.qk_rope_head_dim, - plan_params.page_size, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, - ) + # Number of tokens needed in the KV cache for each sequence after the next pass. + # Kept on the host: every consumer below needs host values, so a device-side computation + # would force a sync per step. These logical block counts describe committed history, not + # all pages that the KV manager has reserved for fixed-width speculative appends. + kv_lens_host = np.asarray(num_cached_tokens_per_seq, + dtype=np.int64) + self.seq_lens_kv.numpy() + logical_num_blocks = ((kv_lens_host + self.page_size - 1) // + self.page_size) + num_blocks = logical_num_blocks + use_full_generation_page_table = ( + self.kv_cache_params.use_full_generation_page_table) + self._uses_full_generation_page_table = use_full_generation_page_table + if use_full_generation_page_table: + # Generation rows need every reserved page to keep a speculative append addressable + # across a page boundary. Treating that wider table as the logical sequence would make + # FlashInfer attend to uncommitted draft slots, so plans and positions use the separate + # device logical lengths populated below. + assert self.request_ids is not None + num_blocks = np.asarray( + _get_page_table_num_blocks( + self.kv_cache_manager, + self.request_ids, + logical_num_blocks.tolist(), + self.num_contexts, + ), + dtype=np.int64, + ) + self.num_blocks = num_blocks.tolist() - @property - def paged_kv_indices(self) -> torch.Tensor: - return self._paged_kv_indices[:self.num_generation_blocks + - self.num_context_blocks] + assert self.request_ids is not None - def get_paged_kv_indices_for_layer(self, layer_idx: int) -> torch.Tensor: - """Return page indices for the pool that *layer_idx* belongs to. + # start and end indices of each sequence in the ragged key and value + # for md attention it's the same as qo_indptr so avoid computing twice. + if self.is_cross: + # start and end indices of each sequence in the ragged key and value + torch.cumsum(self.seq_lens_kv_cuda, + dim=0, + dtype=torch.int32, + out=self._kv_indptr[1:self.seq_lens_kv_cuda.size(0) + + 1]) - For non-VSWA models this returns the default shared indices. - For VSWA models it returns the pool-specific indices. - """ - if (self._vswa_layer_to_pool is None - or self._vswa_pool_indices_cache is None): - return self.paged_kv_indices - pool_id = self._vswa_layer_to_pool.get(layer_idx) - if pool_id is None: - return self.paged_kv_indices - total_blocks = self.num_generation_blocks + self.num_context_blocks - return self._vswa_pool_indices_cache[pool_id][:total_blocks] + # number of cache blocks used by each sequence in the cache + # NOTE: do not use len(block_ids) - that will give you a number + # that can be too big if using chunked prefill/kv cache reuse + # since we allocate all blocks ahead of time. + self.num_context_blocks = sum(self.num_blocks[:self.num_contexts]) + self.num_generation_blocks = sum(self.num_blocks[self.num_contexts:]) - def _sanitize_swa_page_indices(self, page_indices: torch.Tensor, - layer_idx: int) -> None: - """Replace evicted SWA pages with a safe in-range page index.""" - window_vec = getattr(self.kv_cache_manager, 'max_attention_window_vec', - None) - if not window_vec: - return - local_layer_idx = self.kv_cache_manager.layer_offsets[layer_idx] - if window_vec[local_layer_idx] is None: - return + if self._has_mixed_precision_staging(): + self.prepare_mixed_precision_kv( + self.kv_layout, self.kv_cache_manager.tokens_per_block) - # KVCacheManagerV2 marks evicted out-of-window pages with -1. - # FlashInfer may dereference page IDs before applying window_left, so - # keep masked positions in range. The SWA mask excludes their values - # from the attention result. - # - # Page 0 is a live page owned by whichever request holds it, so parking - # there has the kernel dereference a stranger's keys and values, with - # the mask as the only thing keeping them out of the result. When the - # KV cache manager has reserved a guard page (TRTLLM_KV_GUARD_PAGE), - # park on that instead: a page no request can own, holding a known - # pattern. The default -- no guard page -- keeps page 0. - page_indices.masked_fill_(page_indices == BAD_PAGE_INDEX, - self._swa_park_page(layer_idx)) + # indices of used cache blocks for each sequence + primary_layer_idx = self._primary_kv_layer_idx + kv_cache_manager = (self.get_mixed_precision_kv_view(primary_layer_idx) + or self.kv_cache_manager) + paged_kv_indices = kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, self.num_blocks, layer_idx=primary_layer_idx) + if primary_layer_idx is not None: + self._sanitize_swa_page_indices(paged_kv_indices, primary_layer_idx) - def _swa_park_page(self, layer_idx: int) -> int: - """Page index that masked-out entries are pointed at for this layer.""" - guard = getattr(self.kv_cache_manager, 'guard_page_index', None) - if guard is None: - return 0 - page = guard(layer_idx) - return 0 if page is None else int(page) + self._paged_kv_indices[:paged_kv_indices.size(0)].copy_( + paged_kv_indices, non_blocking=True) - def swap_paged_kv_indices_for_layer(self, layer_idx: int) -> None: - """Copy pool-specific page indices into the shared buffer. + # Retain a host mirror of _paged_kv_indices: decode plans build the + # trtllm-gen block tables from host data, with no GPU round trips. + # The VSWA block below re-points the mirror at the active pool's + # copy whenever the device buffer is swapped. + self._host_paged_kv_indices = paged_kv_indices - The FlashInfer wrappers reference ``_paged_kv_indices`` directly, - so we overwrite its contents with the correct pool's data before - each layer's plan/run cycle. We track the currently active pool - so we only copy when the pool actually changes. + # VSWA: build per-pool page index CUDA tensors so each layer can use + # the indices that match its own pool's buffer. Tensors live on CUDA + # so that forward_impl swap via copy_() is device-to-device (CUDA-graph + # capturable). + if self._vswa_layer_to_pool is not None: + unique_pools = set(self._vswa_layer_to_pool.values()) + primary_pool_id = self._primary_kv_pool_id + # Use dedicated pre-allocated buffers for each pool's indices. + # These buffers are created in __post_init__ so their addresses + # stay stable across CUDA-graph replays. + total_idx = paged_kv_indices.size(0) + primary_buf = self._vswa_pool_bufs[primary_pool_id] + primary_buf[:total_idx].copy_(self._paged_kv_indices[:total_idx], + non_blocking=True) + self._vswa_pool_indices_cache = { + primary_pool_id: primary_buf, + } + self._host_pool_indices = {primary_pool_id: paged_kv_indices} + for pool_id in unique_pools: + if pool_id == primary_pool_id: + continue + rep_layer = self._vswa_pool_to_rep_layer[pool_id] + kv_cache_manager = (self.get_mixed_precision_kv_view(rep_layer) + or self.kv_cache_manager) + pool_indices = \ + kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, self.num_blocks, layer_idx=rep_layer) + self._sanitize_swa_page_indices(pool_indices, rep_layer) + buf = self._vswa_pool_bufs[pool_id] + buf[:pool_indices.size(0)].copy_(pool_indices, + non_blocking=True) + self._vswa_pool_indices_cache[pool_id] = buf + self._host_pool_indices[pool_id] = pool_indices + self._vswa_active_pool_id = primary_pool_id - """ - if self._vswa_layer_to_pool is None: - return - pool_id = self._vswa_layer_to_pool.get(layer_idx) - if pool_id is None: - return # Layer not in VSWA mapping - active = getattr(self, '_vswa_active_pool_id', None) - if pool_id == active and not self.is_cuda_graph: - return # Buffer already has the right data - n = self._paged_kv_indices.numel() if self.is_cuda_graph else ( - self.num_generation_blocks + self.num_context_blocks) - src = self._vswa_pool_indices_cache[pool_id][:n] - self._paged_kv_indices[:n].copy_(src, non_blocking=True) - self._vswa_active_pool_id = pool_id - # Keep the host mirror in lockstep with the device buffer so decode - # plans build their block tables from the active pool's indices. - self._host_paged_kv_indices = self._host_pool_indices[pool_id] + # number of tokens in the last cache block used by each sequence, + # derived from the logical rather than reservation-width page count. + paged_kv_last_page_len = _to_int32_tensor(kv_lens_host - + (logical_num_blocks - 1) * + self.page_size) + self._paged_kv_last_page_len[:paged_kv_last_page_len.size(0)].copy_( + paged_kv_last_page_len, non_blocking=True) - @property - def paged_kv_last_page_len(self) -> torch.Tensor: - return self._paged_kv_last_page_len[:self.num_contexts + - self.num_generations] + # Ragged page table, see https://docs.flashinfer.ai/tutorials/kv_layout.html#page-table-layout + # For decoding, this MUST be allocated ahead of time (for CUDA graphs). + # Prefill is prepared here as well just for the sake of consistency. + paged_kv_indptr_decode = _to_int32_tensor( + np.concatenate([[0], np.cumsum(num_blocks[self.num_contexts:])])) + self.paged_kv_indptr_decode[:paged_kv_indptr_decode.size(0)].copy_( + paged_kv_indptr_decode, non_blocking=True) + # Retain the host copy: decode plans hand it to flashinfer so that + # its indptr.cpu()/get_seq_lens calls do no D2H work. + self._host_paged_kv_indptr_decode = paged_kv_indptr_decode - @property - def qo_indptr(self) -> torch.Tensor: - return self._qo_indptr[:self.num_contexts + self.num_generations + 1] + paged_kv_indptr_prefill = _to_int32_tensor( + np.concatenate([[0], np.cumsum(num_blocks[:self.num_contexts])])) + self.paged_kv_indptr_prefill[:paged_kv_indptr_prefill.size(0)].copy_( + paged_kv_indptr_prefill, non_blocking=True) - @property - def kv_indptr(self) -> torch.Tensor: - return self._kv_indptr[:self.num_contexts + self.num_generations + 1] + # This paged_kv_indptr attribute has both prefill and decode information in it. + # It's for the append_paged_kv_cache kernel. + if self.num_contexts == 0: + self.paged_kv_indptr = self.paged_kv_indptr_decode[: + paged_kv_indptr_decode + .size(0)] + elif self.num_generations == 0: + self.paged_kv_indptr = self.paged_kv_indptr_prefill[: + paged_kv_indptr_prefill + .size(0)] + else: + assert not self.is_cuda_graph, "Cannot mix decode/prefill with CUDA graphs" + # Accumulate on the host and stage through pinned memory: .cuda() + # on an unpinned tensor is a synchronous H2D that stalls the + # executor thread behind in-flight kernels on every mixed step. + self.paged_kv_indptr = _to_int32_tensor( + np.concatenate([[0], + np.cumsum(num_blocks)])).to(device='cuda', + non_blocking=True) - @property - def cached_token_lens(self) -> torch.Tensor: - return self._cached_token_lens[:self.num_contexts + - self.num_generations] + # For cross attention, num_tokens is 0 during decode, and we don't need to update kv cache. + if self.num_tokens > 0: + if use_full_generation_page_table: + # The page-table indptr now describes addressable capacity. Deriving positions from + # it would count reserved pages as committed KV and potentially shift appends beyond + # the true sequence end. + logical_kv_lens = _to_int32_tensor(kv_lens_host) + self._logical_kv_lens[:logical_kv_lens.numel()].copy_( + logical_kv_lens, non_blocking=True) + position_kv_lens = self._logical_kv_lens[:self.num_seqs] + else: + position_kv_lens = flashinfer.get_seq_lens( + self.paged_kv_indptr, + self.paged_kv_last_page_len, + self.page_size, + ) + batch_indices, positions = flashinfer.get_batch_indices_positions( + self.kv_indptr, + position_kv_lens, + self.num_tokens, + ) + self._batch_indices[:batch_indices.size(0)].copy_(batch_indices, + non_blocking=True) + self._positions[:positions.size(0)].copy_(positions, + non_blocking=True) - @property - def batch_indices(self) -> torch.Tensor: - return self._batch_indices[:self.num_tokens] + # Re-plan MLA wrappers outside of forward/capture using the params + # cached by prior warmup forwards. Forward still handles first-use or + # dtype/shape changes by syncing only on a plan cache miss. + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False + if (self.num_contexts > 0 and self._mla_ragged_plan_params is not None + and self._ragged_prefill_wrapper is not None): + ragged_indptr = self.qo_indptr[:self.num_contexts + 1] + self._do_plan_ragged(ragged_indptr, ragged_indptr, + self._mla_ragged_plan_params) + self._mla_ragged_planned = True - @property - def positions(self) -> torch.Tensor: - return self._positions[:self.num_tokens] + if (self.num_contexts > 0 and self._mla_context_plan_params is not None + and self._mla_context_wrapper is not None): + num_contexts = self.num_contexts + num_context_blocks = self.num_context_blocks + context_qo_indptr = self.qo_indptr[:num_contexts + 1] + context_kv_indptr = self.paged_kv_indptr_prefill[:num_contexts + 1] + context_kv_indices = self._paged_kv_indices[:num_context_blocks] + context_last_page_len = self._paged_kv_last_page_len[:num_contexts] + self._do_plan_mla_context( + qo_indptr=context_qo_indptr, + kv_indptr=context_kv_indptr, + kv_indices=context_kv_indices, + kv_last_page_len=context_last_page_len, + plan_params=self._mla_context_plan_params, + ) + self._mla_context_planned = True - def get_draft_metadata( - self, - draft_kv_cache_manager: Optional[Any] = None, - ) -> "FlashInferAttentionMetadata": - """Return a planned draft view over shared or separate KV cache.""" - uses_shared_kv_cache = draft_kv_cache_manager is None - if self._draft_metadata is None: - draft_metadata = copy.copy(self) - draft_metadata._is_shared_kv_draft_view = uses_shared_kv_cache - draft_metadata._is_separate_kv_draft_view = ( - not uses_shared_kv_cache) - draft_metadata._draft_metadata = None - draft_metadata.workspace_buffer = self.workspace_buffer - draft_metadata.cuda_graph_buffers = None - draft_metadata.cross = None - if uses_shared_kv_cache: - draft_metadata.max_num_tokens = self.max_num_requests - else: - draft_metadata.kv_cache_manager = draft_kv_cache_manager - draft_metadata._seq_lens = None - draft_metadata._seq_lens_cuda = None - draft_metadata._seq_lens_kv = None - draft_metadata._seq_lens_kv_cuda = None - draft_metadata._saved_tensors = {} - if uses_shared_kv_cache: - draft_metadata.seq_lens = torch.ones((self.max_num_requests, ), - dtype=torch.int) - draft_metadata.num_contexts = 0 - else: - draft_metadata.seq_lens = self.seq_lens - draft_metadata.num_contexts = self.num_contexts - draft_metadata.seq_lens_kv = None - if not uses_shared_kv_cache: - for pool_id in set((self._vswa_layer_to_pool or {}).values()): - setattr(draft_metadata, f"_vswa_pool_buf_{pool_id}", None) - draft_metadata.__post_init__() - if not uses_shared_kv_cache: - # Keep the existing speculative worker's backend-neutral - # kv_lens_cuda protocol scoped to the separate-KV draft view. - draft_metadata.kv_lens_cuda = ( - draft_metadata._draft_kv_runtime_lens) - self._draft_metadata = draft_metadata - draft_metadata._sync_draft_view(self) - elif (self._draft_metadata._is_shared_kv_draft_view - != uses_shared_kv_cache): - raise RuntimeError( - "FlashInfer metadata cannot mix shared and separate draft KV " - "views.") - if not uses_shared_kv_cache: - # The separate view runs on the target's staging copy of the draft manager. - self.sync_draft_mixed_precision_staging() - return self._draft_metadata + # Re-plan the MLA decode wrapper outside of any stream capture. + if (self.num_generations > 0 + and self._mla_decode_plan_params is not None + and self._mla_decode_wrapper is not None): + self._do_plan_mla_decode(self._mla_decode_plan_params) + self._mla_decode_planned = True - def _sync_draft_view(self, target: "FlashInferAttentionMetadata") -> None: - """Refresh a shared- or separate-KV draft metadata view.""" - if not (self._is_shared_kv_draft_view - or self._is_separate_kv_draft_view): - raise RuntimeError("Only a draft metadata view can be synchronized") - - num_seqs = target.num_seqs - self.request_ids = target.request_ids - self.prompt_lens = target.prompt_lens - self.kv_cache_params = target.kv_cache_params - self.all_rank_num_tokens = target.all_rank_num_tokens - - if self._is_separate_kv_draft_view: - self.padded_num_tokens = target.padded_num_tokens - self.seq_lens = target.seq_lens - self.seq_lens_kv = None - self.num_contexts = target.num_contexts - self._uses_full_draft_page_table = False - # Reuse the target's staging copy of a mixed-precision draft manager. - self.mixed_precision_staging = target.draft_mixed_precision_staging - self.draft_mixed_precision_staging = None - self.prepare() - torch.add( - target._cached_token_lens[:num_seqs], - target.seq_lens_kv_cuda[:num_seqs], - out=self._draft_kv_runtime_lens[:num_seqs], - ) - if self.num_contexts == 0: - self._prepare_full_draft_page_table() - return - - self.kv_cache_manager = target.kv_cache_manager - self.seq_lens = torch.ones((num_seqs, ), dtype=torch.int) - self.seq_lens_kv = None - self.num_contexts = 0 - self.num_blocks = list(target.num_blocks) - self.num_context_blocks = 0 - self.num_generation_blocks = sum(self.num_blocks) - self.num_ctx_cached_tokens = 0 - self._multi_item_params = None - - total_blocks = self.num_generation_blocks - self._paged_kv_indices[:total_blocks].copy_( - target._paged_kv_indices[:total_blocks], non_blocking=True) - self._host_paged_kv_indices = target._host_paged_kv_indices - - if (target._vswa_layer_to_pool is not None - and target._vswa_pool_indices_cache is not None): - self._vswa_pool_indices_cache = {} - self._host_pool_indices = dict(target._host_pool_indices) - for pool_id, source in target._vswa_pool_indices_cache.items(): - destination = getattr(self, f"_vswa_pool_buf_{pool_id}") - destination[:total_blocks].copy_(source[:total_blocks], - non_blocking=True) - self._vswa_pool_indices_cache[pool_id] = destination + # VSWA: restore primary pool indices as the default. + if (self._vswa_layer_to_pool is not None + and self._vswa_pool_indices_cache is not None): primary_pool_id = self._primary_kv_pool_id + total_blocks = self.num_generation_blocks + self.num_context_blocks + src = self._vswa_pool_indices_cache[primary_pool_id][:total_blocks] + self._paged_kv_indices[:total_blocks].copy_(src, non_blocking=True) self._vswa_active_pool_id = primary_pool_id - self._host_paged_kv_indices = self._host_pool_indices[ - primary_pool_id] - self._paged_kv_indices[:total_blocks].copy_( - self._vswa_pool_indices_cache[primary_pool_id][:total_blocks], - non_blocking=True) - else: - self._vswa_pool_indices_cache = None - self._host_pool_indices = {} - self._vswa_active_pool_id = None - - host_indptr = maybe_pin_memory( - torch.from_numpy( - np.concatenate([[0], np.cumsum(self.num_blocks) - ]).astype(np.int32, copy=False))) - self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, - non_blocking=True) - self._host_paged_kv_indptr_decode = host_indptr - self.paged_kv_indptr_prefill[0].zero_() - self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] - self._paged_kv_last_page_len[:num_seqs].copy_( - target._paged_kv_last_page_len[:num_seqs], non_blocking=True) - self._cached_token_lens[:num_seqs].copy_( - target._cached_token_lens[:num_seqs], non_blocking=True) + self._host_paged_kv_indices = \ + self._host_pool_indices[primary_pool_id] - host_qo_indptr = torch.arange(num_seqs + 1, - dtype=torch.int32, - pin_memory=prefer_pinned()) - self._qo_indptr[:num_seqs + 1].copy_(host_qo_indptr, non_blocking=True) - full_kv_lens = (target._cached_token_lens[:num_seqs] + - target.seq_lens_kv_cuda[:num_seqs]) - self._draft_kv_runtime_lens[:num_seqs].copy_(full_kv_lens) + self._debug_validate_page_table(kv_lens_host, logical_num_blocks, + num_blocks) - if self.is_cuda_graph: - # Graph replay keeps the captured trtllm-gen plan. Refresh only - # its stable block-table buffers; re-planning the wrapper would - # mutate state owned by the captured kernel. + # CUDA graph + trtllm-gen: update _block_tables and _kv_lens_buffer + # so the trtllm-gen decode kernel uses current page indices. + if (self.is_cuda_graph and self._vswa_layer_to_pool is not None + and self._vswa_pool_indices_cache is not None + and self.num_generations > 0): + decode_blocks = num_blocks[self.num_contexts:] for plan_params, wrappers in self._plan_params_to_wrappers.items(): - self._build_decode_block_tables(plan_params, wrappers) - else: - self._clean_cached_plans(defer_plan=False) + if plan_params.attention_mask_data is not None: + continue + decode_wrapper = wrappers.decode_wrapper + block_tables = getattr(decode_wrapper, '_block_tables', None) + if block_tables is None: + continue + pool_id = plan_params.kv_pool_id + if pool_id is None: + continue + batch_size, table_width = block_tables.shape + rows = min(batch_size, self.num_generations) + num_blocks_per_row = decode_blocks[:rows] + active_width = min(int(num_blocks_per_row.max()), table_width) + # Include the previous live extent once when the batch or its longest sequence + # shrinks. The zero-filled copy below clears entries that are no longer live; the + # next update can shrink. + update_rows = min( + batch_size, + max(rows, wrappers.decode_block_table_active_rows)) + update_width = min( + table_width, + max(active_width, wrappers.decode_block_table_active_width)) + host_block_tables = wrappers.host_decode_block_tables + if (host_block_tables is None + or host_block_tables.size(0) < update_rows + or host_block_tables.size(1) < update_width): + # Grow geometrically to avoid reallocating whenever the longest sequence + # acquires one more KV-cache block. + host_width = min( + max(64, 1 << (update_width - 1).bit_length()), + table_width) + host_block_tables = torch.zeros( + (self.max_num_requests, host_width), + dtype=torch.int32, + pin_memory=prefer_pinned()) + wrappers.host_decode_block_tables = host_block_tables - def update_shared_kv_draft_lengths( - self, - target: "FlashInferAttentionMetadata", - num_accepted_tokens: torch.Tensor, - num_contexts: int, - ) -> None: - """Publish the accepted target prefix to assistant decode wrappers.""" - if not self._is_shared_kv_draft_view: - raise RuntimeError( - "Draft KV lengths can only be set on the shared-KV view") - num_seqs = target.num_seqs - cached_lens = target._cached_token_lens[:num_seqs] - runtime_lens = self._draft_kv_runtime_lens[:num_seqs] - if num_contexts > 0: - runtime_lens[:num_contexts].copy_( - cached_lens[:num_contexts] + - target.seq_lens_kv_cuda[:num_contexts]) - runtime_lens[num_contexts:num_seqs].copy_( - cached_lens[num_contexts:num_seqs] + - num_accepted_tokens[num_contexts:num_seqs]) - self._update_draft_kv_lengths() + # Build only the live (or previously live) rectangle on the host. This preserves + # padded-row zeroing when batch size shrinks without launching full-capacity + # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. + host_table = host_block_tables[:update_rows, :update_width] + host_table.zero_() + host_pool_indices = self._host_pool_indices[pool_id] + # Pool indices are flattened as context blocks followed by each generation request's + # blocks. + source_offset = self.num_context_blocks + # This loop scales with the number of generation requests. Vectorizing ragged + # rows would require padded mask or index tensor, so keep contiguous row copies + # until profiling identifies this as a CPU bottleneck. + for row, num_blocks_for_row in enumerate(num_blocks_per_row): + num_blocks_for_row = int(num_blocks_for_row) + copy_width = min(num_blocks_for_row, table_width) + host_table[row, :copy_width].copy_( + host_pool_indices[source_offset:source_offset + + copy_width]) + # Advance by the uncropped count so the next request starts at the correct + # offset even if this row hit `table_width`. + source_offset += num_blocks_for_row + # Keep the graph-captured device address stable and transfer only the compact + # rectangle prepared in pinned host memory. + block_tables[:update_rows, :update_width].copy_( + host_block_tables[:update_rows, :update_width], + non_blocking=True) + wrappers.decode_block_table_active_rows = rows + wrappers.decode_block_table_active_width = active_width + kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) + if kv_lens_buf is not None: + decode_kv_lens = _to_int32_tensor( + kv_lens_host[self.num_contexts:self.num_contexts + + self.num_generations]) + kv_lens_buf[:self.num_generations].copy_(decode_kv_lens, + non_blocking=True) + if self.num_generations < batch_size: + kv_lens_buf[self.num_generations:batch_size].zero_() - def apply_spec_decode_kv_lens_offsets( - self, - offsets: torch.Tensor, - num_generations: int, - tokens_per_generation: int, - *, - num_chunked_contexts: int = 0, - restore: bool = False, - ) -> None: - """Apply overlap-scheduler corrections to FlashInfer's live KV state.""" - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - raise RuntimeError( - "Speculative KV offsets must be applied to target metadata") - if not self._uses_full_generation_page_table: - return - if num_chunked_contexts == 0 and num_generations == 0: - return + # Refresh captured FA2 schedules only after all page metadata updates. + # Under CUDA graphs every non-trtllm-gen decode wrapper carries + # fa2_plan_num_blocks, so this is what keeps replayed plans current; + # the deferral below only affects wrappers without a recorded tuple. + # Defer ordinary multi-wrapper plans to forward_impl; single-wrapper + # models still plan eagerly because forward_impl cannot plan during + # graph capture. + active_wrappers = [ + pp for pp in self._plan_params_to_wrappers + if pp.attention_mask_data is None + ] + defer_plan = len(active_wrappers) > 1 + if not (self._is_separate_kv_draft_view and self.is_cuda_graph): + self._refresh_fa2_cuda_graph_plans() + self._clean_cached_plans(defer_plan=defer_plan) - direction = -1 if restore else 1 - num_contexts = self.num_contexts - if num_chunked_contexts > 0: - row_slice = slice(num_contexts - num_chunked_contexts, num_contexts) - runtime_offsets = offsets[:num_chunked_contexts] - num_runtime_tokens = num_chunked_contexts * tokens_per_generation - token_slice = slice(self.num_ctx_tokens - num_runtime_tokens, - self.num_ctx_tokens) - else: - row_slice = slice(num_contexts, num_contexts + num_generations) - runtime_offsets = offsets[:num_generations] - num_runtime_tokens = num_generations * tokens_per_generation - token_slice = slice(self.num_ctx_tokens, - self.num_ctx_tokens + num_runtime_tokens) + if (not self._is_shared_kv_draft_view + and not self._is_separate_kv_draft_view + and self._draft_metadata is not None): + self._draft_metadata.core()._sync_draft_view(md) - self._cached_token_lens[row_slice].add_(runtime_offsets, - alpha=direction) - self._logical_kv_lens[row_slice].add_(runtime_offsets, alpha=direction) - token_offsets = runtime_offsets.repeat_interleave(tokens_per_generation) - self._positions[token_slice].add_(token_offsets, alpha=direction) - view = self.mixed_precision_staging - if view is not None and view.manager is self.kv_cache_manager: - # Keep the staging lengths in step with the KV lengths. - view.lengths[:, row_slice].add_(runtime_offsets, alpha=direction) + if md.has_cross_sub_metadata: + md.refresh_cross_query_side().prepare() - for wrappers in self._plan_params_to_wrappers.values(): - self._publish_decode_wrapper_kv_lens(wrappers.decode_wrapper) + def needs_plan(self, plan_params: PlanParams) -> bool: + if plan_params not in self._plan_params_to_wrappers: + return True - def _publish_decode_wrapper_kv_lens(self, decode_wrapper) -> None: - """Publish device-logical lengths to a trtllm-gen decode wrapper.""" - kv_lens_buffer = getattr(decode_wrapper, "_kv_lens_buffer", None) - if kv_lens_buffer is None or self.num_generations == 0: - return + wrappers = self._plan_params_to_wrappers[plan_params] + return not wrappers.is_planned - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - kv_lens_buffer[:self.num_generations].copy_( - self._draft_kv_runtime_lens[:self.num_generations]) - return - if not self._uses_full_generation_page_table: - return - - start = self.num_contexts - end = start + self.num_generations - torch.add( - self._cached_token_lens[start:end], - self.seq_lens_kv_cuda[start:end], - out=kv_lens_buffer[:self.num_generations], - ) + def get_prefill_wrapper( + self, plan_params: PlanParams + ) -> flashinfer.BatchPrefillWithPagedKVCacheWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[plan_params].prefill_wrapper + assert result is not None, "Prefill wrapper was not created in plan()" + return result - def _debug_validate_page_table(self, kv_lens_host, logical_num_blocks, - num_blocks) -> None: - """Diagnostic: check the page table handed to FlashInfer is well formed. + def get_decode_wrapper( + self, plan_params: PlanParams + ) -> flashinfer.BatchDecodeWithPagedKVCacheWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[plan_params].decode_wrapper + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + if result._backend != "trtllm-gen": + raise ValueError( + "FlashInfer draft metadata views require the trtllm-gen " + "decode backend.") + self._publish_decode_wrapper_kv_lens(result) + return result - Every structural claim the kernel relies on is already on the host by - the time this runs -- the index mirrors, the indptr host copies and the - length arrays are all numpy or CPU tensors -- so the check costs no GPU - work and no synchronization, which is what makes it usable on a path - where a synchronization can hide the fault being chased. + def get_ragged_prefill_wrapper( + self, plan_params: PlanParams) -> _RaggedPrefillWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[ + plan_params].ragged_prefill_wrapper + assert result is not None, "Ragged prefill wrapper was not created in plan()" + return result - Off unless ``TRTLLM_FI_PAGE_TABLE_CHECK`` is set. Reports are bounded, - because a malformed row repeats on every step of the request that owns - it, and one line is printed when the check is armed and finds nothing, - so that a clean run is distinguishable from a run the environment - variable never reached. - """ - if not os.environ.get(_PAGE_TABLE_CHECK_ENV, "").strip(): - return - cls = FlashInferAttentionMetadata - if cls._page_table_check_budget <= 0: - return - pool_size = getattr(self.kv_cache_manager, 'blocks_in_primary_pool', - None) - pools = getattr(self, '_host_pool_indices', None) or { - 0: getattr(self, '_host_paged_kv_indices', None) - } - host_pools = { - pool_id: - (indices.cpu().numpy() if torch.is_tensor(indices) else indices) - for pool_id, indices in pools.items() - } - try: - problems = check_page_table( - host_pools, - num_blocks, - logical_num_blocks, - kv_lens_host, - self.page_size, - pool_size=pool_size, + def plan_ragged( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + plan_params: RaggedPlanParams, + ) -> flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper: + """Plan MLA ragged prefill with expanded K, V (not paged).""" + if self._ragged_prefill_wrapper is None: + self._ragged_prefill_wrapper = flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper( + self.workspace_buffer, + "NHD", ) - except Exception as exc: # pragma: no cover - a diagnostic must not break a run - cls._page_table_check_budget -= 1 - logger.warning( - f"[flashinfer] page table check failed to run: {exc}") - return - if problems: - cls._page_table_check_budget -= 1 - logger.warning( - f"[flashinfer] page table check, {self.num_contexts} context / " - f"{self.num_generations} generation rows: " + - "; ".join(problems)) - elif cls._page_table_check_budget == _PAGE_TABLE_CHECK_REPORTS: - cls._page_table_check_budget -= 1 - logger.info( - f"[flashinfer] page table check armed and clean on a batch of " - f"{self.num_contexts} context / {self.num_generations} " - f"generation rows, pool size {pool_size}") - - def _prepare_full_draft_page_table(self) -> None: - """Expose every allocated draft page and use device KV lengths.""" - if self._uses_full_draft_page_table: - return - assert self.request_ids is not None - block_ids_per_seq = self.kv_cache_manager.get_batch_cache_indices( - self.request_ids, layer_idx=self._primary_kv_layer_idx) - self.num_blocks = [len(block_ids) for block_ids in block_ids_per_seq] - self.num_context_blocks = 0 - self.num_generation_blocks = sum(self.num_blocks) - paged_kv_indices = self.kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, - self.num_blocks, - layer_idx=self._primary_kv_layer_idx) - self._paged_kv_indices[:paged_kv_indices.numel()].copy_( - paged_kv_indices, non_blocking=True) - self._host_paged_kv_indices = paged_kv_indices + if self._mla_ragged_plan_params != plan_params: + self._mla_ragged_planned = False + self._mla_ragged_plan_params = plan_params - host_indptr = maybe_pin_memory( - torch.from_numpy( - np.concatenate([[0], np.cumsum(self.num_blocks) - ]).astype(np.int32, copy=False))) - num_seqs = self.num_seqs - self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, - non_blocking=True) - self._host_paged_kv_indptr_decode = host_indptr - self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] - self._uses_full_draft_page_table = True - self._update_draft_kv_lengths() + if self._mla_ragged_planned: + return self._ragged_prefill_wrapper - if self.is_cuda_graph: - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - self._build_decode_block_tables(plan_params, wrappers) - elif not torch.cuda.is_current_stream_capturing(): - self._clean_cached_plans(defer_plan=False) + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise ValueError( + "Cannot plan() flashinfer MLA ragged prefill while the stream " + "is capturing. Make sure prepare() has run at least one " + "warmup forward pass before capture.") - def _update_draft_kv_lengths(self) -> None: - """Publish runtime KV lengths to a shared or separate draft view.""" - num_seqs = self.num_seqs - runtime_lens = self._draft_kv_runtime_lens[:num_seqs] - self._cached_token_lens[:num_seqs].copy_(runtime_lens) - self._paged_kv_last_page_len[:num_seqs].copy_(runtime_lens) - self._paged_kv_last_page_len[:num_seqs].sub_(1) - torch.remainder(self._paged_kv_last_page_len[:num_seqs], - self.page_size, - out=self._paged_kv_last_page_len[:num_seqs]) - self._paged_kv_last_page_len[:num_seqs].add_(1) - if self._is_shared_kv_draft_view: - return + # Split append_paged_mla_kv_cache from plan() when this wrapper needs a + # new plan. Reusing a cached plan avoids this sync on later layers. + torch.cuda.current_stream().synchronize() - self._cached_token_lens[:num_seqs].sub_(1) - self._positions[:num_seqs].copy_(self._cached_token_lens[:num_seqs]) - torch.arange(num_seqs + 1, - dtype=torch.int32, - device=self._qo_indptr.device, - out=self._qo_indptr[:num_seqs + 1]) - torch.arange(num_seqs, - dtype=torch.int32, - device=self._batch_indices.device, - out=self._batch_indices[:num_seqs]) + self._do_plan_ragged(qo_indptr, kv_indptr, plan_params) + self._mla_ragged_planned = True - def update_for_spec_dec(self) -> None: - if not self._is_separate_kv_draft_view: - return - self._prepare_full_draft_page_table() - self._update_draft_kv_lengths() + return self._ragged_prefill_wrapper - def __post_init__(self) -> None: - super().__post_init__() - self._post_init_with_buffers(self.cuda_graph_buffers) + def _do_plan_ragged( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + plan_params: RaggedPlanParams, + ) -> None: + assert self._ragged_prefill_wrapper is not None + self._ragged_prefill_wrapper.plan( + qo_indptr, + kv_indptr, + plan_params.num_heads, + plan_params.num_kv_heads, + plan_params.head_dim, + head_dim_vo=plan_params.head_dim_vo, + use_fp16_qk_reduction=False, + causal=True, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + sm_scale=plan_params.sm_scale, + ) - def _post_init_with_buffers(self, buffers) -> None: - capture_graph = self.is_cuda_graph + def plan_mla_decode( + self, + plan_params: MLAPlanParams, + ) -> object: + """Plan MLA decode using BatchMLAPagedAttentionWrapper. - if self.workspace_buffer is None: - # Note: even though flashinfer only recommends 128 MB, we have to push it - # a bit higher to cover all possible CUDA graph cases. If it's too small, - # warmup will crash. - # FlashInfer's split-KV decode plan sizes its temp buffers proportionally to - # SM count; 320 MB was tuned for a 148-SM baseline, so scale up (never down) - # on GPUs with more SMs. - baseline_sms = 148 - baseline_bytes = 320 * 1024 * 1024 - num_sms = torch.cuda.get_device_properties( - torch.cuda.current_device()).multi_processor_count - workspace_bytes = max(baseline_bytes, - baseline_bytes * num_sms // baseline_sms) - self.workspace_buffer = self.get_empty( - buffers, - (workspace_bytes, ), - dtype=torch.uint8, - cache_name="workspace_buffer", - capture_graph=capture_graph, + Caches the wrapper and plan params; the actual plan() call is driven + by prepare() so it runs outside of CUDA graph capture. + """ + if self._mla_decode_wrapper is None: + self._mla_decode_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=self.is_cuda_graph, + qo_indptr=self._mla_qo_indptr_buf, + kv_indptr=self.paged_kv_indptr_decode, + kv_indices=self._paged_kv_indices, + kv_len_arr=self._mla_kv_len_arr_buf, + backend="auto", ) - self.paged_kv_indptr_decode = torch.empty((self.max_num_requests + 1, ), - device='cuda', - dtype=torch.int) - self.paged_kv_indptr_prefill = torch.empty( - (self.max_num_requests + 1, ), device='cuda', dtype=torch.int) - self._paged_kv_last_page_len = torch.empty((self.max_num_requests, ), - device='cuda', - dtype=torch.int) + if self._mla_decode_plan_params != plan_params: + self._mla_decode_planned = False - self._qo_indptr = torch.zeros(self.max_num_requests + 1, - device='cuda', - dtype=torch.int) + # Cache params so prepare() can re-plan on subsequent forward passes. + self._mla_decode_plan_params = plan_params - self._kv_indptr = torch.zeros( - self.max_num_requests + 1, device='cuda', - dtype=torch.int) if self.is_cross else self._qo_indptr + if self._mla_decode_planned: + return self._mla_decode_wrapper - self._ragged_qo_indptr_buf = self.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_ragged_qo_indptr_buf", - capture_graph=capture_graph, - ) - self._ragged_kv_indptr_buf = self.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_ragged_kv_indptr_buf", - capture_graph=capture_graph, - ) + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise ValueError( + "Cannot plan() flashinfer MLA decode while the stream is " + "capturing. Make sure prepare() has run at least one warmup " + "forward pass before capture.") - self._cached_token_lens = torch.empty((self.max_num_requests, ), - dtype=torch.int, - device='cuda') - self._logical_kv_lens = self.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int, - cache_name="_logical_kv_lens", - capture_graph=capture_graph, - ) - self._draft_kv_runtime_lens = self.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int, - cache_name="_draft_kv_runtime_lens", - capture_graph=capture_graph, - ) - self._batch_indices = torch.empty((self.max_num_tokens, ), - dtype=torch.int, - device='cuda') - self._positions = torch.empty((self.max_num_tokens, ), - dtype=torch.int, - device='cuda') - self._plan_params_to_wrappers = {} + # Split append_paged_mla_kv_cache from plan() on a cache miss. prepare() + # calls _do_plan_mla_decode() directly and does not need this sync. + torch.cuda.current_stream().synchronize() + self._do_plan_mla_decode(plan_params) + self._mla_decode_planned = True + return self._mla_decode_wrapper - # Host-side state for sync-free trtllm-gen decode plans, retained by - # prepare(): a host mirror of _paged_kv_indices (kept in lockstep - # with the device buffer, with per-pool copies for VSWA swaps) and a - # host copy of the decode indptr. The per-plan_params persistent - # block-table buffers live on FlashInferWrappers. - self._host_pool_indices: Dict[int, torch.Tensor] = {} - self._host_paged_kv_indices: Optional[torch.Tensor] = None - self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None - self._uses_full_generation_page_table = False - self._max_num_blocks_per_seq = 0 + def plan_mla_context( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_last_page_len: torch.Tensor, + plan_params: MLAPlanParams, + ) -> object: + """Plan MLA context with cached KV using BatchMLAPagedAttentionWrapper.""" + if self._mla_context_wrapper is None: + self._mla_context_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=False, + backend="auto", + ) - # VSWA (Variable Sliding Window Attention): models with per-layer - # max_attention_window create separate V2 pool groups with independent - # page numbering. We need per-pool paged_kv_indices so each layer can - # use the indices that match its pool's buffer. - self._vswa_layer_to_pool: Optional[Dict[int, int]] = None - self._vswa_pool_indices_cache: Optional[Dict[int, torch.Tensor]] = None - self._primary_kv_layer_idx: Optional[int] = None - self._primary_kv_pool_id = 0 + if self._mla_context_plan_params != plan_params: + self._mla_context_planned = False + self._mla_context_plan_params = plan_params - if self.kv_cache_manager is not None: - blocks_in_primary_pool = self.kv_cache_manager.blocks_in_primary_pool - attention_layer_indices = _get_attention_layer_indices( - self.kv_cache_manager) + if self._mla_context_planned: + return self._mla_context_wrapper - # Maximum block count across attention pools: sizes the shared - # page-index buffer and the VSWA pool buffers below. - max_num_blocks = blocks_in_primary_pool - for layer_idx in attention_layer_indices: - layer_buffer = self.kv_cache_manager.get_buffers(layer_idx) - if layer_buffer is not None: - max_num_blocks = max(max_num_blocks, layer_buffer.shape[0]) - if self._has_mixed_precision_staging(): - # The staging page space spans every sequence slot. - max_num_blocks = max( - max_num_blocks, - (self.max_num_sequences or self.max_num_requests) * - self.kv_cache_manager.max_blocks_per_seq) - self._paged_kv_indices = self.get_empty( - buffers, - (max_num_blocks, ), - dtype=torch.int, - cache_name="_paged_kv_indices", - capture_graph=capture_graph, - ) - self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise ValueError( + "Cannot plan() flashinfer MLA context while the stream is " + "capturing. Make sure prepare() has run at least one warmup " + "forward pass before capture.") - # Layers may share one page-index list only when they are in the - # same pool, page-index scale, and KV factor: VSWA splits pools, - # and per-layer geometry (e.g. Gemma4 sliding/global head_dim) - # splits scales even when all windows collapse to max_seq_len and - # is_vswa is False. Mixed K-only/KV layers can use different view - # indices even with the same physical scale. Guarded on V2-specific - # attributes (V1 managers lack the per-pool infrastructure). - mgr = self.kv_cache_manager - get_scale = getattr(mgr, 'get_layer_page_index_scale', None) - get_kv_factor = getattr(mgr, 'get_layer_kv_factor', None) - layer_space: Dict[int, int] = {} - if hasattr(mgr, 'layer_to_pool_mapping_dict') and get_scale: - space_ids = {} - for layer_idx in attention_layer_indices: - layer_offset = mgr.layer_offsets[layer_idx] - key = (mgr.layer_to_pool_mapping_dict[layer_offset], - get_scale(layer_idx), - get_kv_factor(layer_idx) if get_kv_factor else None) - if (self._has_mixed_precision_staging() and layer_offset - in mgr.mixed_precision_high_layers): - key = ("mixed_precision_staging", ) - space_ids.setdefault(key, len(space_ids)) - layer_space[layer_idx] = space_ids[key] - if layer_space: - self._primary_kv_layer_idx = next(iter(layer_space)) - self._primary_kv_pool_id = layer_space[ - self._primary_kv_layer_idx] - if layer_space and (getattr(mgr, 'is_vswa', False) - or len(set(layer_space.values())) > 1): - self._vswa_layer_to_pool = {} - self._vswa_pool_to_rep_layer: Dict[int, int] = {} - for layer_idx, pool_id in layer_space.items(): - self._vswa_layer_to_pool[layer_idx] = pool_id - if pool_id not in self._vswa_pool_to_rep_layer: - self._vswa_pool_to_rep_layer[pool_id] = layer_idx - # Pre-allocate VSWA pool cache buffers. These must be - # stable (never reallocated) so that CUDA-graph-recorded - # copies reference valid addresses across replays. - # max_num_blocks (computed above) covers ALL pools so that - # secondary pool buffers are large enough. - for pool_id in set(self._vswa_layer_to_pool.values()): - buf_key = f'_vswa_pool_buf_{pool_id}' - if getattr(self, buf_key, None) is None: - setattr( - self, buf_key, - torch.empty(max_num_blocks, - dtype=torch.int, - device='cuda')) - # Stable buffers for FlashInfer MLA decode; required for CUDA graphs. - self._mla_qo_indptr_buf = self.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_mla_qo_indptr_buf", - capture_graph=capture_graph, - ) - self._mla_kv_len_arr_buf = self.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int32, - cache_name="_mla_kv_len_arr_buf", - capture_graph=capture_graph, - ) - # Rebind the wrapper to the freshly allocated buffers. - self._ragged_prefill_wrapper = None - self._mla_decode_wrapper = None - self._mla_context_wrapper = None - self._mla_ragged_plan_params = None - self._mla_context_plan_params = None - self._mla_decode_plan_params = None - self._mla_ragged_planned = False - self._mla_context_planned = False - self._mla_decode_planned = False - self._multi_item_params = None + # Split append_paged_mla_kv_cache from plan() when this wrapper needs a + # new plan. Reusing a cached plan avoids this sync on later layers. + torch.cuda.current_stream().synchronize() - def create_cuda_graph_metadata(self, - max_batch_size: int, - sub_cross_metadata: bool = False, - max_draft_tokens: int = 0, - buffers=None, - encode_only: bool = False) -> Self: - metadata = super().create_cuda_graph_metadata(max_batch_size, - sub_cross_metadata, - max_draft_tokens, buffers, - encode_only) - metadata.max_num_requests = max_batch_size - metadata.max_num_tokens = max_batch_size * (1 + max_draft_tokens) - # The graph owns distinct assistant wrappers and stable metadata - # buffers; never inherit the eager view through the shallow copy. - metadata._draft_metadata = None - # Post init again to make sure all tensors are allocated - metadata.__post_init__() - return metadata + self._do_plan_mla_context( + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + plan_params, + ) + self._mla_context_planned = True - @property - def page_size(self) -> int: - """ - Number of tokens per cache page - """ - assert self.kv_cache_manager is not None, ( - "page_size is undefined without a KV cache manager; use the " - "ragged prefill path instead.") - return self.kv_cache_manager.tokens_per_block + return self._mla_context_wrapper - def _plan_ragged_no_kv( + def _do_plan_mla_context( self, - plan_params: PlanParams, - ragged_prefill_wrapper: Any, - flashinfer_backend: str, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_last_page_len: torch.Tensor, + plan_params: MLAPlanParams, ) -> None: - is_causal = plan_params.attention_mask_type == AttentionMaskType.causal - if plan_params.attention_mask_data is not None: - window_left = -1 - else: - window_left = plan_params.window_left + assert self._mla_context_wrapper is not None - # Lengths are already on GPU via AttentionMetadata (seq_lens setter -> _seq_lens_cuda). - assert self.seq_lens_cuda is not None - assert self.seq_lens is not None + num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] + kv_len_arr = (num_pages_per_seq - + 1) * plan_params.page_size + kv_last_page_len - # NB: When kv_cache_manager is None (e.g. ViT), ragged prefill runs only for the context phase. - # Restrict seq_lens and index tensors to the first num_contexts entries accordingly. - # - # NB: flashinfer expects seq_len (BatchPrefillWithPagedKVCacheWrapper only), paged_kv_last_page_len, qo_indptr, - # and paged_kv_indptr to be host tensors (if not, it synchronously copies them to the host). - # => not using self.qo_indptr and self.kv_indptr here - num_contexts = self.num_contexts - q_and_kv_seqlens = self.seq_lens[:num_contexts] - q_and_kv_seqlens_cuda = self.seq_lens_cuda[:self.num_contexts] - query_output_indptr = torch.zeros( - num_contexts + 1, - dtype=torch.int32, - pin_memory=prefer_pinned(), + self._mla_context_wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + plan_params.num_heads, + plan_params.kv_lora_rank, + plan_params.qk_rope_head_dim, + plan_params.page_size, + causal=True, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + sm_scale=plan_params.sm_scale, ) - key_value_indptr = query_output_indptr - num_query_output_heads = plan_params.num_heads - num_key_value_heads = plan_params.num_kv_heads - attention_head_dim = plan_params.head_dim - max_query_tokens_per_sequence = 0 - if num_contexts > 0: - query_output_indptr[1:].copy_(torch.cumsum(q_and_kv_seqlens, dim=0)) - # key_value_indptr is query_output_indptr - max_query_tokens_per_sequence = cast(int, - q_and_kv_seqlens.max().item()) + def _do_plan_mla_decode(self, plan_params: MLAPlanParams) -> None: + """Compute MLA decode plan inputs and call wrapper.plan(). - if (multi_item_params := plan_params.multi_item_params) is not None: - if flashinfer_backend not in ["fa2", "fa3"]: - raise ValueError( - "The selected FlashInfer attention backend does not support multi-item attention masking." - ) - assert isinstance(ragged_prefill_wrapper, - flashinfer.BatchPrefillWithPagedKVCacheWrapper - ) # _PageSizeOnePrefillWrapper - plan_kwargs = dict( - prefix_len_ptr=multi_item_params.prefix_len_ptr, - token_pos_in_items_ptr=multi_item_params.token_pos_in_items_ptr, - token_pos_in_items_len=multi_item_params.token_pos_in_items_len, - max_item_len_ptr=multi_item_params.max_item_len_ptr, - paged_kv_indptr=key_value_indptr, - paged_kv_indices=torch.arange(key_value_indptr[-1].item(), - device=key_value_indptr.device, - dtype=key_value_indptr.dtype), - paged_kv_last_page_len=torch.ones_like(q_and_kv_seqlens), - page_size=1, - seq_lens=q_and_kv_seqlens, - ) + Must run outside of CUDA graph capture. + """ + num_gen = self.num_generations + kv_indptr = self.paged_kv_indptr_decode[:num_gen + 1] + kv_indices = self._paged_kv_indices[self.num_context_blocks:self. + num_context_blocks + + self.num_generation_blocks] + kv_last_page = self._paged_kv_last_page_len[self.num_contexts:self. + num_contexts + num_gen] + + # _qo_indptr is ordered [context_seqs..., generation_seqs...]; rebase + # the generation slice to 0. + num_ctx = self.num_contexts + qo_indptr = self._qo_indptr[num_ctx:num_ctx + num_gen + + 1] - self._qo_indptr[num_ctx] + + if self._uses_full_generation_page_table: + kv_len_arr = self._logical_kv_lens[num_ctx:num_ctx + num_gen] else: - assert isinstance(ragged_prefill_wrapper, - flashinfer.BatchPrefillWithRaggedKVCacheWrapper) - max_key_value_tokens_per_sequence = max_query_tokens_per_sequence - # Do not pass v_indptr/o_indptr for self-attention. When they are omitted, FlashInfer - # aliases them to the CUDA-graph-stable kv/qo buffers; passing them explicitly creates - # fresh CUDA tensors during plan(), leaving replay with stale captured offset pointers. - plan_kwargs = dict( - kv_indptr=key_value_indptr, - seq_lens=q_and_kv_seqlens_cuda, - seq_lens_q=q_and_kv_seqlens_cuda, - max_sequence_kv= - max_key_value_tokens_per_sequence, # breaks BatchPrefillWithPagedKVCacheWrapper - ) - ragged_prefill_wrapper.plan( - qo_indptr=query_output_indptr, - num_qo_heads=num_query_output_heads, - num_kv_heads=num_key_value_heads, - head_dim_qk=attention_head_dim, - custom_mask=plan_params.attention_mask_data, - causal=is_causal, - sm_scale=plan_params.sm_scale, - window_left=window_left, + num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] + kv_len_arr = (num_pages_per_seq - + 1) * plan_params.page_size + kv_last_page + + self._mla_decode_wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + plan_params.num_heads, + plan_params.kv_lora_rank, + plan_params.qk_rope_head_dim, + plan_params.page_size, + causal=True, q_data_type=plan_params.q_dtype, kv_data_type=plan_params.kv_dtype, - max_token_per_sequence=max_query_tokens_per_sequence, - **plan_kwargs, + sm_scale=plan_params.sm_scale, ) - def _process_multi_item_part_lens( - self, - multi_item_part_lens: list[list[int]], - *, - device: torch.device, - ) -> FlashInferMultiItemParams: - if self.num_generations > 0: - raise ValueError( - "\"multi_item_part_lens\" not supported for generation requests." - ) - if len(multi_item_part_lens) != self.num_contexts: - raise ValueError( - "\"multi_item_part_lens\" needs to be provided for all requests." - ) - if any([len(part_lens) < 2 for part_lens in multi_item_part_lens]): - raise ValueError( - "\"multi_item_part_lens\" must have at least two elements") + @property + def paged_kv_indices(self) -> torch.Tensor: + return self._paged_kv_indices[:self.num_generation_blocks + + self.num_context_blocks] - prefix_len_ptr = torch.tensor( - [req_part_lens[0] for req_part_lens in multi_item_part_lens], - pin_memory=prefer_pinned(), - dtype=torch.uint32, - ).to(device=device, non_blocking=True) - token_pos_in_items_raw_lens = [ # 'raw' lengths before padding - sum(req_part_lens[1:]) + len(req_part_lens) - for req_part_lens in multi_item_part_lens - ] - token_pos_in_items_len = max(token_pos_in_items_raw_lens) - max_item_len_ptr = torch.tensor( - [max(req_part_lens[1:]) for req_part_lens in multi_item_part_lens], - pin_memory=prefer_pinned(), - dtype=torch.uint16, - ).to(device=device, non_blocking=True) + def get_paged_kv_indices_for_layer(self, layer_idx: int) -> torch.Tensor: + """Return page indices for the pool that *layer_idx* belongs to. - # token_pos_in_items_ptr is obtained by concatenating range(item_len + 1) for each item in - # every request, followed by [0] (final delimiter) which is fused with padding for simplicity. - range_ends = torch.tensor( - [ - item_len + 1 - for req_part_lens, token_pos_in_items_raw_len in zip( - multi_item_part_lens, - token_pos_in_items_raw_lens, - strict=True) for item_len in - chain(req_part_lens[1:], - [token_pos_in_items_len - token_pos_in_items_raw_len]) - ], - pin_memory=prefer_pinned(), - dtype=torch.int32, - ).to(device=device, non_blocking=True) - token_pos_in_items_ptr = torch_multi_arange( - range_ends, - output_length=(token_pos_in_items_len * len(multi_item_part_lens)), - ) - # next, mask out the padding - mask_entries = torch.arange(2, dtype=torch.uint8).to( - device=device, - non_blocking=True, - dtype=torch.bool, - ).repeat(len(multi_item_part_lens)) # NB: .expand() does not work here - mask_entry_repeats = torch.tensor( - [ - repeat - for token_pos_in_items_raw_len in token_pos_in_items_raw_lens - for repeat in [ - token_pos_in_items_raw_len, - token_pos_in_items_len - token_pos_in_items_raw_len, - ] - ], - pin_memory=prefer_pinned(), - dtype=torch.int32, - ).to(device=device, non_blocking=True) - padding_mask = torch.repeat_interleave( - input=mask_entries, - repeats=mask_entry_repeats, - output_size=token_pos_in_items_ptr.size(0), - ) - token_pos_in_items_ptr.masked_fill_(padding_mask, 0) - token_pos_in_items_ptr = token_pos_in_items_ptr.to(dtype=torch.uint16, - non_blocking=True) + For non-VSWA models this returns the default shared indices. + For VSWA models it returns the pool-specific indices. + """ + if (self._vswa_layer_to_pool is None + or self._vswa_pool_indices_cache is None): + return self.paged_kv_indices + pool_id = self._vswa_layer_to_pool.get(layer_idx) + if pool_id is None: + return self.paged_kv_indices + total_blocks = self.num_generation_blocks + self.num_context_blocks + return self._vswa_pool_indices_cache[pool_id][:total_blocks] - return FlashInferMultiItemParams( - prefix_len_ptr=prefix_len_ptr, - max_item_len_ptr=max_item_len_ptr, - token_pos_in_items_ptr=token_pos_in_items_ptr, - token_pos_in_items_len=token_pos_in_items_len, - ) + def _sanitize_swa_page_indices(self, page_indices: torch.Tensor, + layer_idx: int) -> None: + """Replace evicted SWA pages with a safe in-range page index.""" + window_vec = getattr(self.kv_cache_manager, 'max_attention_window_vec', + None) + if not window_vec: + return + local_layer_idx = self.kv_cache_manager.layer_offsets[layer_idx] + if window_vec[local_layer_idx] is None: + return - def _build_decode_block_tables( - self, plan_params: PlanParams, - wrappers: FlashInferWrappers) -> Optional[torch.Tensor]: - """Build the trtllm-gen decode block table on the host. + # KVCacheManagerV2 marks evicted out-of-window pages with -1. + # FlashInfer may dereference page IDs before applying window_left, so + # keep masked positions in range. The SWA mask excludes their values + # from the attention result. + # + # Page 0 is a live page owned by whichever request holds it, so parking + # there has the kernel dereference a stranger's keys and values, with + # the mask as the only thing keeping them out of the result. When the + # KV cache manager has reserved a guard page (TRTLLM_KV_GUARD_PAGE), + # park on that instead: a page no request can own, holding a known + # pattern. The default -- no guard page -- keeps page 0. + page_indices.masked_fill_(page_indices == BAD_PAGE_INDEX, + self._swa_park_page(layer_idx)) - When ``block_tables`` is not passed to - ``BatchDecodeWithPagedKVCacheWrapper.plan()``, flashinfer rebuilds - it with a per-request loop whose slice bounds are GPU scalars — - one cudaStreamSynchronize + one scalar D2H read per generation - request per plan. Instead, build the ``[num_gens, max_n]`` table - here with one vectorized pass over the host mirror of the active - pool's flat page indices (no GPU reads) and push it with a single - async H2D into a persistent device buffer held by ``wrappers``. - Under CUDA-graph metadata the buffer is allocated once at full - capacity width and never moves, so captured decode kernels keep - reading valid memory while prepare() refreshes the contents in - place; the eager path re-plans every step and may grow its buffer - geometrically. + def _swa_park_page(self, layer_idx: int) -> int: + """Page index that masked-out entries are pointed at for this layer.""" + guard = getattr(self.kv_cache_manager, 'guard_page_index', None) + if guard is None: + return 0 + page = guard(layer_idx) + return 0 if page is None else int(page) + + def swap_paged_kv_indices_for_layer(self, layer_idx: int) -> None: + """Copy pool-specific page indices into the shared buffer. + + The FlashInfer wrappers reference ``_paged_kv_indices`` directly, + so we overwrite its contents with the correct pool's data before + each layer's plan/run cycle. We track the currently active pool + so we only copy when the pool actually changes. - Returns None when the host data (or memory for the buffer) is - unavailable; the caller then falls back to flashinfer's own - rebuild — itself sync-free now that the plan indptr is a host - tensor. """ - if plan_params.attention_mask_data is not None: - # Masked plans are flushed every step, taking their wrappers - # (and any buffers on them) along; don't churn per-step - # block-table allocations for them. - return None - num_gens = self.num_generations - if num_gens == 0: - return None - host_paged_kv_indices = self._host_pool_indices.get( - plan_params.kv_pool_id, self._host_paged_kv_indices) - if host_paged_kv_indices is None: - return None - gen_num_blocks = np.asarray(self.num_blocks[self.num_contexts:], - dtype=np.int64) - max_n = int(gen_num_blocks.max()) - if max_n > self._max_num_blocks_per_seq: - # A request can never reference more than max_blocks_per_seq; - # defensive guard for inconsistent metadata. - return None - block_tables = wrappers.decode_block_tables - if (self.is_cuda_graph and block_tables is not None - and block_tables.size(1) < max_n): - # Never reallocate under CUDA graphs: captured decode kernels - # hold the buffer address. - return None - if block_tables is None or block_tables.size(1) < max_n: - if self.is_cuda_graph: - # Allocated once at capture warmup; full capacity width so - # replays never need a wider table. - width = self._max_num_blocks_per_seq - else: - # Eager path replans (and re-reads the table) every step, - # so the buffer may grow geometrically as sequences do. - width = min(max(64, 1 << (max_n - 1).bit_length()), - self._max_num_blocks_per_seq) - try: - block_tables = torch.zeros((self.max_num_requests, width), - dtype=torch.int32, - device='cuda') - except torch.OutOfMemoryError: - # E.g. the KV-estimation warmup forwards run with device - # memory deliberately exhausted; fall back to flashinfer's - # rebuild rather than failing the forward. - return None - wrappers.decode_block_tables = block_tables - host_block_tables = wrappers.host_decode_block_tables - if host_block_tables is None or host_block_tables.size(1) < max_n: - host_width = min(max(64, 1 << (max_n - 1).bit_length()), - block_tables.size(1)) - host_block_tables = torch.zeros((self.max_num_requests, host_width), - dtype=torch.int32, - pin_memory=prefer_pinned()) - wrappers.host_decode_block_tables = host_block_tables - start = self.num_context_blocks - decode_flat = host_paged_kv_indices.numpy()[start:start + - int(gen_num_blocks.sum())] - table = host_block_tables.numpy()[:num_gens, :max_n] - table[:] = 0 - table[np.arange(max_n)[None, :] < gen_num_blocks[:, None]] = \ - decode_flat - # Rewriting the host buffer is safe: _plan_with_params synchronizes - # the stream before planning, so the previous plan's H2D has - # completed. - block_tables[:num_gens, :max_n].copy_( - host_block_tables[:num_gens, :max_n], non_blocking=True) - # Seed the extent used by `prepare()` to clear entries if the first graph replay has fewer - # requests or a narrower block table. - wrappers.decode_block_table_active_rows = num_gens - wrappers.decode_block_table_active_width = max_n - return block_tables[:num_gens] + if self._vswa_layer_to_pool is None: + return + pool_id = self._vswa_layer_to_pool.get(layer_idx) + if pool_id is None: + return # Layer not in VSWA mapping + active = getattr(self, '_vswa_active_pool_id', None) + if pool_id == active and not self.is_cuda_graph: + return # Buffer already has the right data + n = self._paged_kv_indices.numel() if self.is_cuda_graph else ( + self.num_generation_blocks + self.num_context_blocks) + src = self._vswa_pool_indices_cache[pool_id][:n] + self._paged_kv_indices[:n].copy_(src, non_blocking=True) + self._vswa_active_pool_id = pool_id + # Keep the host mirror in lockstep with the device buffer so decode + # plans build their block tables from the active pool's indices. + self._host_paged_kv_indices = self._host_pool_indices[pool_id] - def _clean_cached_plans(self, *, defer_plan: bool): - for plan_params in list(self._plan_params_to_wrappers.keys()): - # Generally, plan_params with non-trivial attention masking are relevant only the - # corresponding forward pass. So, flush them out here as they won't be relevant for - # subsequent forward calls. - if plan_params.attention_mask_data is None and plan_params.multi_item_params is None: - wrappers = self._plan_params_to_wrappers[plan_params] - if wrappers.fa2_plan_num_blocks is not None: - continue - wrappers.is_planned = False - if not defer_plan: - self._plan_with_params(plan_params) - else: - del self._plan_params_to_wrappers[plan_params] + @property + def paged_kv_last_page_len(self) -> torch.Tensor: + return self._paged_kv_last_page_len[:self.num_contexts + + self.num_generations] - def _refresh_fa2_cuda_graph_plans(self) -> None: - """Refresh captured FA2 schedules after page metadata is finalized.""" - num_blocks = tuple(self.num_blocks[self.num_contexts:]) - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - if (plan_params.attention_mask_data is not None - or plan_params.multi_item_params is not None - or wrappers.fa2_plan_num_blocks is None): - continue - if not num_blocks: - wrappers.fa2_plan_num_blocks = None - elif wrappers.fa2_plan_num_blocks != num_blocks: - # Graph replay does not re-enter forward_impl. Each wrapper - # owns its persistent integer plan workspace, while the shared - # float workspace is run scratch. - wrappers.is_planned = False - self._plan_with_params(plan_params) + @property + def qo_indptr(self) -> torch.Tensor: + return self._qo_indptr[:self.num_contexts + self.num_generations + 1] - def prepare(self) -> None: + @property + def kv_indptr(self) -> torch.Tensor: + return self._kv_indptr[:self.num_contexts + self.num_generations + 1] - def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: - """Host int32 staging tensor for async H2D copies; pinned when - beneficial. + @property + def cached_token_lens(self) -> torch.Tensor: + return self._cached_token_lens[:self.num_contexts + + self.num_generations] - torch.tensor(..., pin_memory=True) rejects numpy sources, hence - from_numpy + pin. - """ - return maybe_pin_memory( - torch.from_numpy(arr.astype(np.int32, copy=False))) + @property + def batch_indices(self) -> torch.Tensor: + return self._batch_indices[:self.num_tokens] - super().prepare() - extra_attrs = get_model_extra_attrs() - if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(self) - # start and end indices of each sequence in the ragged query - assert self.seq_lens_cuda is not None - torch.cumsum(self.seq_lens_cuda, - dim=0, - dtype=torch.int32, - out=self._qo_indptr[1:self.seq_lens_cuda.size(0) + 1]) + @property + def positions(self) -> torch.Tensor: + return self._positions[:self.num_tokens] - if self.multi_item_part_lens is not None: - self._multi_item_params = self._process_multi_item_part_lens( - self.multi_item_part_lens, device=self.seq_lens_cuda.device) - else: - self._multi_item_params = None + def resync_length_derived_state(self) -> None: + """Refresh a separate-KV draft view after a speculative sub-step. - if self.kv_cache_manager is None: - assert self.request_ids is not None - assert self.num_generations == 0, ( - "FlashInfer without a KV cache manager only supports context-only " - "batches (num_generations == 0) in TRT-LLM.") - if self.is_cross: - raise NotImplementedError( - "FlashInfer without a KV cache manager is not tested for cross attention." - ) - self.kv_cache_params = KVCacheParams(use_cache=False) - n = self.num_seqs - self._cached_token_lens[:n].zero_() - self.num_ctx_cached_tokens = 0 - self._clean_cached_plans(defer_plan=False) + FlashInfer replans from the lengths on its next forward, so only a + draft view over a separate KV cache has state to refresh here. + """ + if not self._is_separate_kv_draft_view: return + self._prepare_full_draft_page_table() + self._update_draft_kv_lengths() - if self._multi_item_params is not None: - raise ValueError( - "multi_item_part_lens with KV cache is not supported") - - # number of tokens in the kv cache for each sequence in the batch - num_cached_tokens_per_seq = self.kv_cache_params.num_cached_tokens_per_seq - cached_token_lens = torch.tensor(num_cached_tokens_per_seq, - dtype=torch.int, - pin_memory=prefer_pinned()) - self._cached_token_lens[:cached_token_lens.size(0)].copy_( - cached_token_lens, non_blocking=True) - if self.num_contexts > 0: - self.num_ctx_cached_tokens = sum( - num_cached_tokens_per_seq[:self.num_contexts]) - else: - self.num_ctx_cached_tokens = 0 + def cuda_graph_copy_changes(self, max_batch_size: int, + max_draft_tokens: int) -> Dict[str, Any]: + # The graph's wrappers and buffers are sized for the graph's batch. + return { + "max_num_requests": max_batch_size, + "max_num_tokens": max_batch_size * (1 + max_draft_tokens), + } - # Number of tokens needed in the KV cache for each sequence after the next pass. - # Kept on the host: every consumer below needs host values, so a device-side computation - # would force a sync per step. These logical block counts describe committed history, not - # all pages that the KV manager has reserved for fixed-width speculative appends. - kv_lens_host = np.asarray(num_cached_tokens_per_seq, - dtype=np.int64) + self.seq_lens_kv.numpy() - logical_num_blocks = ((kv_lens_host + self.page_size - 1) // - self.page_size) - num_blocks = logical_num_blocks - use_full_generation_page_table = ( - self.kv_cache_params.use_full_generation_page_table) - self._uses_full_generation_page_table = use_full_generation_page_table - if use_full_generation_page_table: - # Generation rows need every reserved page to keep a speculative append addressable - # across a page boundary. Treating that wider table as the logical sequence would make - # FlashInfer attend to uncommitted draft slots, so plans and positions use the separate - # device logical lengths populated below. - assert self.request_ids is not None - num_blocks = np.asarray( - _get_page_table_num_blocks( - self.kv_cache_manager, - self.request_ids, - logical_num_blocks.tolist(), - self.num_contexts, - ), - dtype=np.int64, - ) - self.num_blocks = num_blocks.tolist() + def get_draft_metadata( + self, + draft_kv_cache_manager: Optional[Any] = None, + ) -> "FlashInferAttentionMetadata": + """Return a planned draft view over shared or separate KV cache.""" + md = self.metadata + uses_shared_kv_cache = draft_kv_cache_manager is None + if self._draft_metadata is None: + # A metadata of its own over the same configuration: a shared-KV + # view plans one query token per request over the target's cache; + # a separate-KV view addresses the draft manager's pools. Both run + # on the target's workspace. + if uses_shared_kv_cache: + draft_metadata = md.constructed_copy( + cuda_graph_buffers=None, + max_num_tokens=self.max_num_requests, + seq_lens=torch.ones((self.max_num_requests, ), + dtype=torch.int), + num_contexts=0, + seq_lens_kv=None, + cross=None, + ) + else: + draft_metadata = md.constructed_copy( + share=False, + cuda_graph_buffers=None, + kv_cache_manager=draft_kv_cache_manager, + workspace_buffer=self.workspace_buffer, + seq_lens=self.seq_lens, + num_contexts=self.num_contexts, + seq_lens_kv=None, + cross=None, + ) + # The view's core is this class's, so this class sets it up. + draft_core = draft_metadata.core() + draft_core._is_shared_kv_draft_view = uses_shared_kv_cache + draft_core._is_separate_kv_draft_view = not uses_shared_kv_cache + if not uses_shared_kv_cache: + # Keep the existing speculative worker's backend-neutral + # kv_lens_cuda protocol scoped to the separate-KV draft view. + draft_core.kv_lens_cuda = draft_core._draft_kv_runtime_lens + self._draft_metadata = draft_metadata + draft_metadata.core()._sync_draft_view(md) + elif (self._draft_metadata.core()._is_shared_kv_draft_view + != uses_shared_kv_cache): + raise RuntimeError( + "FlashInfer metadata cannot mix shared and separate draft KV " + "views.") + if not uses_shared_kv_cache: + # The separate view's staging copy of the draft manager holds the + # same tokens as the target's: take the lengths the overlap + # scheduler corrected on the device since prepare. + self._draft_metadata.core()._follow_staging_lengths( + self.mixed_precision_staging) + return self._draft_metadata - assert self.request_ids is not None + def _follow_staging_lengths( + self, source: Optional["MixedPrecisionKVStaging"]) -> None: + """Copy ``source``'s staging lengths into this view's staging copy.""" + view = self.mixed_precision_staging + if (view is not None and source is not None and view.lengths is not None + and source.lengths is not None): + view.lengths.copy_(source.lengths) - # start and end indices of each sequence in the ragged key and value - # for self attention it's the same as qo_indptr so avoid computing twice. - if self.is_cross: - # start and end indices of each sequence in the ragged key and value - torch.cumsum(self.seq_lens_kv_cuda, - dim=0, - dtype=torch.int32, - out=self._kv_indptr[1:self.seq_lens_kv_cuda.size(0) + - 1]) + def _mixed_precision_staging_name(self) -> str: + # A separate-KV draft view stages the draft manager. + if self._is_separate_kv_draft_view: + return "draft_mixed" + return super()._mixed_precision_staging_name() - # number of cache blocks used by each sequence in the cache - # NOTE: do not use len(block_ids) - that will give you a number - # that can be too big if using chunked prefill/kv cache reuse - # since we allocate all blocks ahead of time. - self.num_context_blocks = sum(self.num_blocks[:self.num_contexts]) - self.num_generation_blocks = sum(self.num_blocks[self.num_contexts:]) + def _sync_draft_view(self, target: "FlashInferAttentionMetadata") -> None: + """Refresh a shared- or separate-KV draft metadata view.""" + md = self.metadata + if not (self._is_shared_kv_draft_view + or self._is_separate_kv_draft_view): + raise RuntimeError("Only a draft metadata view can be synchronized") - if self._has_mixed_precision_staging(): - self.prepare_mixed_precision_kv( - self.kv_layout, self.kv_cache_manager.tokens_per_block) + num_seqs = target.num_seqs + self.request_ids = target.request_ids + self.prompt_lens = target.prompt_lens + self.kv_cache_params = target.kv_cache_params + self.all_rank_num_tokens = target.all_rank_num_tokens - # indices of used cache blocks for each sequence - primary_layer_idx = self._primary_kv_layer_idx - kv_cache_manager = (self.get_mixed_precision_kv_view(primary_layer_idx) - or self.kv_cache_manager) - paged_kv_indices = kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, self.num_blocks, layer_idx=primary_layer_idx) - if primary_layer_idx is not None: - self._sanitize_swa_page_indices(paged_kv_indices, primary_layer_idx) + if self._is_separate_kv_draft_view: + self.padded_num_tokens = target.padded_num_tokens + self.seq_lens = target.seq_lens + self.seq_lens_kv = None + self.num_contexts = target.num_contexts + self._uses_full_draft_page_table = False + md.prepare() + torch.add( + target.core()._cached_token_lens[:num_seqs], + target.seq_lens_kv_cuda[:num_seqs], + out=self._draft_kv_runtime_lens[:num_seqs], + ) + if self.num_contexts == 0: + self._prepare_full_draft_page_table() + return - self._paged_kv_indices[:paged_kv_indices.size(0)].copy_( - paged_kv_indices, non_blocking=True) + self.kv_cache_manager = target.kv_cache_manager + self.seq_lens = torch.ones((num_seqs, ), dtype=torch.int) + self.seq_lens_kv = None + self.num_contexts = 0 + self.num_blocks = list(target.num_blocks) + self.num_context_blocks = 0 + self.num_generation_blocks = sum(self.num_blocks) + self.num_ctx_cached_tokens = 0 + self._multi_item_params = None - # Retain a host mirror of _paged_kv_indices: decode plans build the - # trtllm-gen block tables from host data, with no GPU round trips. - # The VSWA block below re-points the mirror at the active pool's - # copy whenever the device buffer is swapped. - self._host_paged_kv_indices = paged_kv_indices + total_blocks = self.num_generation_blocks + self._paged_kv_indices[:total_blocks].copy_( + target.core()._paged_kv_indices[:total_blocks], non_blocking=True) + self._host_paged_kv_indices = target.core()._host_paged_kv_indices - # VSWA: build per-pool page index CUDA tensors so each layer can use - # the indices that match its own pool's buffer. Tensors live on CUDA - # so that forward_impl swap via copy_() is device-to-device (CUDA-graph - # capturable). - if self._vswa_layer_to_pool is not None: - unique_pools = set(self._vswa_layer_to_pool.values()) - primary_pool_id = self._primary_kv_pool_id - # Use dedicated pre-allocated buffers for each pool's indices. - # These buffers are created in __post_init__ so their addresses - # stay stable across CUDA-graph replays. - total_idx = paged_kv_indices.size(0) - primary_buf = getattr(self, f'_vswa_pool_buf_{primary_pool_id}') - primary_buf[:total_idx].copy_(self._paged_kv_indices[:total_idx], - non_blocking=True) - self._vswa_pool_indices_cache = { - primary_pool_id: primary_buf, - } - self._host_pool_indices = {primary_pool_id: paged_kv_indices} - for pool_id in unique_pools: - if pool_id == primary_pool_id: - continue - rep_layer = self._vswa_pool_to_rep_layer[pool_id] - kv_cache_manager = (self.get_mixed_precision_kv_view(rep_layer) - or self.kv_cache_manager) - pool_indices = \ - kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, self.num_blocks, layer_idx=rep_layer) - self._sanitize_swa_page_indices(pool_indices, rep_layer) - buf = getattr(self, f'_vswa_pool_buf_{pool_id}') - buf[:pool_indices.size(0)].copy_(pool_indices, + if (target.core()._vswa_layer_to_pool is not None + and target.core()._vswa_pool_indices_cache is not None): + self._vswa_pool_indices_cache = {} + self._host_pool_indices = dict(target.core()._host_pool_indices) + for pool_id, source in target.core()._vswa_pool_indices_cache.items( + ): + destination = self._vswa_pool_bufs[pool_id] + destination[:total_blocks].copy_(source[:total_blocks], non_blocking=True) - self._vswa_pool_indices_cache[pool_id] = buf - self._host_pool_indices[pool_id] = pool_indices + self._vswa_pool_indices_cache[pool_id] = destination + primary_pool_id = self._primary_kv_pool_id self._vswa_active_pool_id = primary_pool_id + self._host_paged_kv_indices = self._host_pool_indices[ + primary_pool_id] + self._paged_kv_indices[:total_blocks].copy_( + self._vswa_pool_indices_cache[primary_pool_id][:total_blocks], + non_blocking=True) + else: + self._vswa_pool_indices_cache = None + self._host_pool_indices = {} + self._vswa_active_pool_id = None - # number of tokens in the last cache block used by each sequence, - # derived from the logical rather than reservation-width page count. - paged_kv_last_page_len = _to_int32_tensor(kv_lens_host - - (logical_num_blocks - 1) * - self.page_size) - self._paged_kv_last_page_len[:paged_kv_last_page_len.size(0)].copy_( - paged_kv_last_page_len, non_blocking=True) - - # Ragged page table, see https://docs.flashinfer.ai/tutorials/kv_layout.html#page-table-layout - # For decoding, this MUST be allocated ahead of time (for CUDA graphs). - # Prefill is prepared here as well just for the sake of consistency. - paged_kv_indptr_decode = _to_int32_tensor( - np.concatenate([[0], np.cumsum(num_blocks[self.num_contexts:])])) - self.paged_kv_indptr_decode[:paged_kv_indptr_decode.size(0)].copy_( - paged_kv_indptr_decode, non_blocking=True) - # Retain the host copy: decode plans hand it to flashinfer so that - # its indptr.cpu()/get_seq_lens calls do no D2H work. - self._host_paged_kv_indptr_decode = paged_kv_indptr_decode + host_indptr = maybe_pin_memory( + torch.from_numpy( + np.concatenate([[0], np.cumsum(self.num_blocks) + ]).astype(np.int32, copy=False))) + self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, + non_blocking=True) + self._host_paged_kv_indptr_decode = host_indptr + self.paged_kv_indptr_prefill[0].zero_() + self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] + self._paged_kv_last_page_len[:num_seqs].copy_( + target.core()._paged_kv_last_page_len[:num_seqs], non_blocking=True) + self._cached_token_lens[:num_seqs].copy_( + target.core()._cached_token_lens[:num_seqs], non_blocking=True) - paged_kv_indptr_prefill = _to_int32_tensor( - np.concatenate([[0], np.cumsum(num_blocks[:self.num_contexts])])) - self.paged_kv_indptr_prefill[:paged_kv_indptr_prefill.size(0)].copy_( - paged_kv_indptr_prefill, non_blocking=True) + host_qo_indptr = torch.arange(num_seqs + 1, + dtype=torch.int32, + pin_memory=prefer_pinned()) + self._qo_indptr[:num_seqs + 1].copy_(host_qo_indptr, non_blocking=True) + full_kv_lens = (target.core()._cached_token_lens[:num_seqs] + + target.seq_lens_kv_cuda[:num_seqs]) + self._draft_kv_runtime_lens[:num_seqs].copy_(full_kv_lens) - # This paged_kv_indptr attribute has both prefill and decode information in it. - # It's for the append_paged_kv_cache kernel. - if self.num_contexts == 0: - self.paged_kv_indptr = self.paged_kv_indptr_decode[: - paged_kv_indptr_decode - .size(0)] - elif self.num_generations == 0: - self.paged_kv_indptr = self.paged_kv_indptr_prefill[: - paged_kv_indptr_prefill - .size(0)] + if self.is_cuda_graph: + # Graph replay keeps the captured trtllm-gen plan. Refresh only + # its stable block-table buffers; re-planning the wrapper would + # mutate state owned by the captured kernel. + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + self._build_decode_block_tables(plan_params, wrappers) else: - assert not self.is_cuda_graph, "Cannot mix decode/prefill with CUDA graphs" - # Accumulate on the host and stage through pinned memory: .cuda() - # on an unpinned tensor is a synchronous H2D that stalls the - # executor thread behind in-flight kernels on every mixed step. - self.paged_kv_indptr = _to_int32_tensor( - np.concatenate([[0], - np.cumsum(num_blocks)])).to(device='cuda', - non_blocking=True) - - # For cross attention, num_tokens is 0 during decode, and we don't need to update kv cache. - if self.num_tokens > 0: - if use_full_generation_page_table: - # The page-table indptr now describes addressable capacity. Deriving positions from - # it would count reserved pages as committed KV and potentially shift appends beyond - # the true sequence end. - logical_kv_lens = _to_int32_tensor(kv_lens_host) - self._logical_kv_lens[:logical_kv_lens.numel()].copy_( - logical_kv_lens, non_blocking=True) - position_kv_lens = self._logical_kv_lens[:self.num_seqs] - else: - position_kv_lens = flashinfer.get_seq_lens( - self.paged_kv_indptr, - self.paged_kv_last_page_len, - self.page_size, - ) - batch_indices, positions = flashinfer.get_batch_indices_positions( - self.kv_indptr, - position_kv_lens, - self.num_tokens, - ) - self._batch_indices[:batch_indices.size(0)].copy_(batch_indices, - non_blocking=True) - self._positions[:positions.size(0)].copy_(positions, - non_blocking=True) + self._clean_cached_plans(defer_plan=False) - # Re-plan MLA wrappers outside of forward/capture using the params - # cached by prior warmup forwards. Forward still handles first-use or - # dtype/shape changes by syncing only on a plan cache miss. - self._mla_ragged_planned = False - self._mla_context_planned = False - self._mla_decode_planned = False - if (self.num_contexts > 0 and self._mla_ragged_plan_params is not None - and self._ragged_prefill_wrapper is not None): - ragged_indptr = self.qo_indptr[:self.num_contexts + 1] - self._do_plan_ragged(ragged_indptr, ragged_indptr, - self._mla_ragged_plan_params) - self._mla_ragged_planned = True + def update_shared_kv_draft_lengths( + self, + target: "FlashInferAttentionMetadata", + num_accepted_tokens: torch.Tensor, + num_contexts: int, + ) -> None: + """Publish the accepted target prefix to assistant decode wrappers.""" + if not self._is_shared_kv_draft_view: + raise RuntimeError( + "Draft KV lengths can only be set on the shared-KV view") + num_seqs = target.num_seqs + cached_lens = target.core()._cached_token_lens[:num_seqs] + runtime_lens = self._draft_kv_runtime_lens[:num_seqs] + if num_contexts > 0: + runtime_lens[:num_contexts].copy_( + cached_lens[:num_contexts] + + target.seq_lens_kv_cuda[:num_contexts]) + runtime_lens[num_contexts:num_seqs].copy_( + cached_lens[num_contexts:num_seqs] + + num_accepted_tokens[num_contexts:num_seqs]) + self._update_draft_kv_lengths() - if (self.num_contexts > 0 and self._mla_context_plan_params is not None - and self._mla_context_wrapper is not None): - num_contexts = self.num_contexts - num_context_blocks = self.num_context_blocks - context_qo_indptr = self.qo_indptr[:num_contexts + 1] - context_kv_indptr = self.paged_kv_indptr_prefill[:num_contexts + 1] - context_kv_indices = self._paged_kv_indices[:num_context_blocks] - context_last_page_len = self._paged_kv_last_page_len[:num_contexts] - self._do_plan_mla_context( - qo_indptr=context_qo_indptr, - kv_indptr=context_kv_indptr, - kv_indices=context_kv_indices, - kv_last_page_len=context_last_page_len, - plan_params=self._mla_context_plan_params, - ) - self._mla_context_planned = True + def apply_spec_decode_kv_lens_offsets( + self, + offsets: torch.Tensor, + num_generations: int, + tokens_per_generation: int, + *, + num_chunked_contexts: int = 0, + restore: bool = False, + ) -> None: + """Apply overlap-scheduler corrections to FlashInfer's live KV state.""" + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + raise RuntimeError( + "Speculative KV offsets must be applied to target metadata") + if not self._uses_full_generation_page_table: + return + if num_chunked_contexts == 0 and num_generations == 0: + return - # Re-plan the MLA decode wrapper outside of any stream capture. - if (self.num_generations > 0 - and self._mla_decode_plan_params is not None - and self._mla_decode_wrapper is not None): - self._do_plan_mla_decode(self._mla_decode_plan_params) - self._mla_decode_planned = True + direction = -1 if restore else 1 + num_contexts = self.num_contexts + if num_chunked_contexts > 0: + row_slice = slice(num_contexts - num_chunked_contexts, num_contexts) + runtime_offsets = offsets[:num_chunked_contexts] + num_runtime_tokens = num_chunked_contexts * tokens_per_generation + token_slice = slice(self.num_ctx_tokens - num_runtime_tokens, + self.num_ctx_tokens) + else: + row_slice = slice(num_contexts, num_contexts + num_generations) + runtime_offsets = offsets[:num_generations] + num_runtime_tokens = num_generations * tokens_per_generation + token_slice = slice(self.num_ctx_tokens, + self.num_ctx_tokens + num_runtime_tokens) - # VSWA: restore primary pool indices as the default. - if (self._vswa_layer_to_pool is not None - and self._vswa_pool_indices_cache is not None): - primary_pool_id = self._primary_kv_pool_id - total_blocks = self.num_generation_blocks + self.num_context_blocks - src = self._vswa_pool_indices_cache[primary_pool_id][:total_blocks] - self._paged_kv_indices[:total_blocks].copy_(src, non_blocking=True) - self._vswa_active_pool_id = primary_pool_id - self._host_paged_kv_indices = \ - self._host_pool_indices[primary_pool_id] + self._cached_token_lens[row_slice].add_(runtime_offsets, + alpha=direction) + self._logical_kv_lens[row_slice].add_(runtime_offsets, alpha=direction) + token_offsets = runtime_offsets.repeat_interleave(tokens_per_generation) + self._positions[token_slice].add_(token_offsets, alpha=direction) + view = self.mixed_precision_staging + if view is not None and view.manager is self.kv_cache_manager: + # Keep the staging lengths in step with the KV lengths. + view.lengths[:, row_slice].add_(runtime_offsets, alpha=direction) + + for wrappers in self._plan_params_to_wrappers.values(): + self._publish_decode_wrapper_kv_lens(wrappers.decode_wrapper) + + def _publish_decode_wrapper_kv_lens(self, decode_wrapper) -> None: + """Publish device-logical lengths to a trtllm-gen decode wrapper.""" + kv_lens_buffer = getattr(decode_wrapper, "_kv_lens_buffer", None) + if kv_lens_buffer is None or self.num_generations == 0: + return + + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + kv_lens_buffer[:self.num_generations].copy_( + self._draft_kv_runtime_lens[:self.num_generations]) + return + if not self._uses_full_generation_page_table: + return + + start = self.num_contexts + end = start + self.num_generations + torch.add( + self._cached_token_lens[start:end], + self.seq_lens_kv_cuda[start:end], + out=kv_lens_buffer[:self.num_generations], + ) + + def _debug_validate_page_table(self, kv_lens_host, logical_num_blocks, + num_blocks) -> None: + """Diagnostic: check the page table handed to FlashInfer is well formed. + + Every structural claim the kernel relies on is already on the host by + the time this runs -- the index mirrors, the indptr host copies and the + length arrays are all numpy or CPU tensors -- so the check costs no GPU + work and no synchronization, which is what makes it usable on a path + where a synchronization can hide the fault being chased. + + Off unless ``TRTLLM_FI_PAGE_TABLE_CHECK`` is set. Reports are bounded, + because a malformed row repeats on every step of the request that owns + it, and one line is printed when the check is armed and finds nothing, + so that a clean run is distinguishable from a run the environment + variable never reached. + """ + if not os.environ.get(_PAGE_TABLE_CHECK_ENV, "").strip(): + return + cls = FlashInferAttentionMetadata + if cls._page_table_check_budget <= 0: + return + pool_size = getattr(self.kv_cache_manager, 'blocks_in_primary_pool', + None) + pools = getattr(self, '_host_pool_indices', None) or { + 0: getattr(self, '_host_paged_kv_indices', None) + } + host_pools = { + pool_id: + (indices.cpu().numpy() if torch.is_tensor(indices) else indices) + for pool_id, indices in pools.items() + } + try: + problems = check_page_table( + host_pools, + num_blocks, + logical_num_blocks, + kv_lens_host, + self.page_size, + pool_size=pool_size, + ) + except Exception as exc: # pragma: no cover - a diagnostic must not break a run + cls._page_table_check_budget -= 1 + logger.warning( + f"[flashinfer] page table check failed to run: {exc}") + return + if problems: + cls._page_table_check_budget -= 1 + logger.warning( + f"[flashinfer] page table check, {self.num_contexts} context / " + f"{self.num_generations} generation rows: " + + "; ".join(problems)) + elif cls._page_table_check_budget == _PAGE_TABLE_CHECK_REPORTS: + cls._page_table_check_budget -= 1 + logger.info( + f"[flashinfer] page table check armed and clean on a batch of " + f"{self.num_contexts} context / {self.num_generations} " + f"generation rows, pool size {pool_size}") + + def _prepare_full_draft_page_table(self) -> None: + """Expose every allocated draft page and use device KV lengths.""" + if self._uses_full_draft_page_table: + return + assert self.request_ids is not None + block_ids_per_seq = self.kv_cache_manager.get_batch_cache_indices( + self.request_ids, layer_idx=self._primary_kv_layer_idx) + self.num_blocks = [len(block_ids) for block_ids in block_ids_per_seq] + self.num_context_blocks = 0 + self.num_generation_blocks = sum(self.num_blocks) + paged_kv_indices = self.kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, + self.num_blocks, + layer_idx=self._primary_kv_layer_idx) + self._paged_kv_indices[:paged_kv_indices.numel()].copy_( + paged_kv_indices, non_blocking=True) + self._host_paged_kv_indices = paged_kv_indices + + host_indptr = maybe_pin_memory( + torch.from_numpy( + np.concatenate([[0], np.cumsum(self.num_blocks) + ]).astype(np.int32, copy=False))) + num_seqs = self.num_seqs + self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, + non_blocking=True) + self._host_paged_kv_indptr_decode = host_indptr + self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] + self._uses_full_draft_page_table = True + self._update_draft_kv_lengths() + + if self.is_cuda_graph: + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + self._build_decode_block_tables(plan_params, wrappers) + elif not torch.cuda.is_current_stream_capturing(): + self._clean_cached_plans(defer_plan=False) + + def _update_draft_kv_lengths(self) -> None: + """Publish runtime KV lengths to a shared or separate draft view.""" + num_seqs = self.num_seqs + runtime_lens = self._draft_kv_runtime_lens[:num_seqs] + self._cached_token_lens[:num_seqs].copy_(runtime_lens) + self._paged_kv_last_page_len[:num_seqs].copy_(runtime_lens) + self._paged_kv_last_page_len[:num_seqs].sub_(1) + torch.remainder(self._paged_kv_last_page_len[:num_seqs], + self.page_size, + out=self._paged_kv_last_page_len[:num_seqs]) + self._paged_kv_last_page_len[:num_seqs].add_(1) + if self._is_shared_kv_draft_view: + return + + self._cached_token_lens[:num_seqs].sub_(1) + self._positions[:num_seqs].copy_(self._cached_token_lens[:num_seqs]) + torch.arange(num_seqs + 1, + dtype=torch.int32, + device=self._qo_indptr.device, + out=self._qo_indptr[:num_seqs + 1]) + torch.arange(num_seqs, + dtype=torch.int32, + device=self._batch_indices.device, + out=self._batch_indices[:num_seqs]) + + @property + def page_size(self) -> int: + """ + Number of tokens per cache page + """ + assert self.kv_cache_manager is not None, ( + "page_size is undefined without a KV cache manager; use the " + "ragged prefill path instead.") + return self.kv_cache_manager.tokens_per_block + + def _plan_ragged_no_kv( + self, + plan_params: PlanParams, + ragged_prefill_wrapper: Any, + flashinfer_backend: str, + ) -> None: + is_causal = plan_params.attention_mask_type == AttentionMaskType.causal + if plan_params.attention_mask_data is not None: + window_left = -1 + else: + window_left = plan_params.window_left + + # Lengths are already on GPU via AttentionMetadata (seq_lens setter -> _seq_lens_cuda). + assert self.seq_lens_cuda is not None + assert self.seq_lens is not None + + # NB: When kv_cache_manager is None (e.g. ViT), ragged prefill runs only for the context phase. + # Restrict seq_lens and index tensors to the first num_contexts entries accordingly. + # + # NB: flashinfer expects seq_len (BatchPrefillWithPagedKVCacheWrapper only), paged_kv_last_page_len, qo_indptr, + # and paged_kv_indptr to be host tensors (if not, it synchronously copies them to the host). + # => not using self.qo_indptr and self.kv_indptr here + num_contexts = self.num_contexts + q_and_kv_seqlens = self.seq_lens[:num_contexts] + q_and_kv_seqlens_cuda = self.seq_lens_cuda[:self.num_contexts] + query_output_indptr = torch.zeros( + num_contexts + 1, + dtype=torch.int32, + pin_memory=prefer_pinned(), + ) + key_value_indptr = query_output_indptr + num_query_output_heads = plan_params.num_heads + num_key_value_heads = plan_params.num_kv_heads + attention_head_dim = plan_params.head_dim + + max_query_tokens_per_sequence = 0 + if num_contexts > 0: + query_output_indptr[1:].copy_(torch.cumsum(q_and_kv_seqlens, dim=0)) + # key_value_indptr is query_output_indptr + max_query_tokens_per_sequence = cast(int, + q_and_kv_seqlens.max().item()) + + if (multi_item_params := plan_params.multi_item_params) is not None: + if flashinfer_backend not in ["fa2", "fa3"]: + raise ValueError( + "The selected FlashInfer attention backend does not support multi-item attention masking." + ) + assert isinstance(ragged_prefill_wrapper, + flashinfer.BatchPrefillWithPagedKVCacheWrapper + ) # _PageSizeOnePrefillWrapper + plan_kwargs = dict( + prefix_len_ptr=multi_item_params.prefix_len_ptr, + token_pos_in_items_ptr=multi_item_params.token_pos_in_items_ptr, + token_pos_in_items_len=multi_item_params.token_pos_in_items_len, + max_item_len_ptr=multi_item_params.max_item_len_ptr, + paged_kv_indptr=key_value_indptr, + paged_kv_indices=torch.arange(key_value_indptr[-1].item(), + device=key_value_indptr.device, + dtype=key_value_indptr.dtype), + paged_kv_last_page_len=torch.ones_like(q_and_kv_seqlens), + page_size=1, + seq_lens=q_and_kv_seqlens, + ) + else: + assert isinstance(ragged_prefill_wrapper, + flashinfer.BatchPrefillWithRaggedKVCacheWrapper) + max_key_value_tokens_per_sequence = max_query_tokens_per_sequence + # Do not pass v_indptr/o_indptr for self-attention. When they are omitted, FlashInfer + # aliases them to the CUDA-graph-stable kv/qo buffers; passing them explicitly creates + # fresh CUDA tensors during plan(), leaving replay with stale captured offset pointers. + plan_kwargs = dict( + kv_indptr=key_value_indptr, + seq_lens=q_and_kv_seqlens_cuda, + seq_lens_q=q_and_kv_seqlens_cuda, + max_sequence_kv= + max_key_value_tokens_per_sequence, # breaks BatchPrefillWithPagedKVCacheWrapper + ) + ragged_prefill_wrapper.plan( + qo_indptr=query_output_indptr, + num_qo_heads=num_query_output_heads, + num_kv_heads=num_key_value_heads, + head_dim_qk=attention_head_dim, + custom_mask=plan_params.attention_mask_data, + causal=is_causal, + sm_scale=plan_params.sm_scale, + window_left=window_left, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + max_token_per_sequence=max_query_tokens_per_sequence, + **plan_kwargs, + ) + + def _process_multi_item_part_lens( + self, + multi_item_part_lens: list[list[int]], + *, + device: torch.device, + ) -> FlashInferMultiItemParams: + if self.num_generations > 0: + raise ValueError( + "\"multi_item_part_lens\" not supported for generation requests." + ) + if len(multi_item_part_lens) != self.num_contexts: + raise ValueError( + "\"multi_item_part_lens\" needs to be provided for all requests." + ) + if any([len(part_lens) < 2 for part_lens in multi_item_part_lens]): + raise ValueError( + "\"multi_item_part_lens\" must have at least two elements") + + prefix_len_ptr = torch.tensor( + [req_part_lens[0] for req_part_lens in multi_item_part_lens], + pin_memory=prefer_pinned(), + dtype=torch.uint32, + ).to(device=device, non_blocking=True) + token_pos_in_items_raw_lens = [ # 'raw' lengths before padding + sum(req_part_lens[1:]) + len(req_part_lens) + for req_part_lens in multi_item_part_lens + ] + token_pos_in_items_len = max(token_pos_in_items_raw_lens) + max_item_len_ptr = torch.tensor( + [max(req_part_lens[1:]) for req_part_lens in multi_item_part_lens], + pin_memory=prefer_pinned(), + dtype=torch.uint16, + ).to(device=device, non_blocking=True) + + # token_pos_in_items_ptr is obtained by concatenating range(item_len + 1) for each item in + # every request, followed by [0] (final delimiter) which is fused with padding for simplicity. + range_ends = torch.tensor( + [ + item_len + 1 + for req_part_lens, token_pos_in_items_raw_len in zip( + multi_item_part_lens, + token_pos_in_items_raw_lens, + strict=True) for item_len in + chain(req_part_lens[1:], + [token_pos_in_items_len - token_pos_in_items_raw_len]) + ], + pin_memory=prefer_pinned(), + dtype=torch.int32, + ).to(device=device, non_blocking=True) + token_pos_in_items_ptr = torch_multi_arange( + range_ends, + output_length=(token_pos_in_items_len * len(multi_item_part_lens)), + ) + # next, mask out the padding + mask_entries = torch.arange(2, dtype=torch.uint8).to( + device=device, + non_blocking=True, + dtype=torch.bool, + ).repeat(len(multi_item_part_lens)) # NB: .expand() does not work here + mask_entry_repeats = torch.tensor( + [ + repeat + for token_pos_in_items_raw_len in token_pos_in_items_raw_lens + for repeat in [ + token_pos_in_items_raw_len, + token_pos_in_items_len - token_pos_in_items_raw_len, + ] + ], + pin_memory=prefer_pinned(), + dtype=torch.int32, + ).to(device=device, non_blocking=True) + padding_mask = torch.repeat_interleave( + input=mask_entries, + repeats=mask_entry_repeats, + output_size=token_pos_in_items_ptr.size(0), + ) + token_pos_in_items_ptr.masked_fill_(padding_mask, 0) + token_pos_in_items_ptr = token_pos_in_items_ptr.to(dtype=torch.uint16, + non_blocking=True) - self._debug_validate_page_table(kv_lens_host, logical_num_blocks, - num_blocks) + return FlashInferMultiItemParams( + prefix_len_ptr=prefix_len_ptr, + max_item_len_ptr=max_item_len_ptr, + token_pos_in_items_ptr=token_pos_in_items_ptr, + token_pos_in_items_len=token_pos_in_items_len, + ) - # CUDA graph + trtllm-gen: update _block_tables and _kv_lens_buffer - # so the trtllm-gen decode kernel uses current page indices. - if (self.is_cuda_graph and self._vswa_layer_to_pool is not None - and self._vswa_pool_indices_cache is not None - and self.num_generations > 0): - decode_blocks = num_blocks[self.num_contexts:] - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - if plan_params.attention_mask_data is not None: - continue - decode_wrapper = wrappers.decode_wrapper - block_tables = getattr(decode_wrapper, '_block_tables', None) - if block_tables is None: - continue - pool_id = plan_params.kv_pool_id - if pool_id is None: - continue - batch_size, table_width = block_tables.shape - rows = min(batch_size, self.num_generations) - num_blocks_per_row = decode_blocks[:rows] - active_width = min(int(num_blocks_per_row.max()), table_width) - # Include the previous live extent once when the batch or its longest sequence - # shrinks. The zero-filled copy below clears entries that are no longer live; the - # next update can shrink. - update_rows = min( - batch_size, - max(rows, wrappers.decode_block_table_active_rows)) - update_width = min( - table_width, - max(active_width, wrappers.decode_block_table_active_width)) - host_block_tables = wrappers.host_decode_block_tables - if (host_block_tables is None - or host_block_tables.size(0) < update_rows - or host_block_tables.size(1) < update_width): - # Grow geometrically to avoid reallocating whenever the longest sequence - # acquires one more KV-cache block. - host_width = min( - max(64, 1 << (update_width - 1).bit_length()), - table_width) - host_block_tables = torch.zeros( - (self.max_num_requests, host_width), - dtype=torch.int32, - pin_memory=prefer_pinned()) - wrappers.host_decode_block_tables = host_block_tables + def _build_decode_block_tables( + self, plan_params: PlanParams, + wrappers: FlashInferWrappers) -> Optional[torch.Tensor]: + """Build the trtllm-gen decode block table on the host. - # Build only the live (or previously live) rectangle on the host. This preserves - # padded-row zeroing when batch size shrinks without launching full-capacity - # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. - host_table = host_block_tables[:update_rows, :update_width] - host_table.zero_() - host_pool_indices = self._host_pool_indices[pool_id] - # Pool indices are flattened as context blocks followed by each generation request's - # blocks. - source_offset = self.num_context_blocks - # This loop scales with the number of generation requests. Vectorizing ragged - # rows would require padded mask or index tensor, so keep contiguous row copies - # until profiling identifies this as a CPU bottleneck. - for row, num_blocks_for_row in enumerate(num_blocks_per_row): - num_blocks_for_row = int(num_blocks_for_row) - copy_width = min(num_blocks_for_row, table_width) - host_table[row, :copy_width].copy_( - host_pool_indices[source_offset:source_offset + - copy_width]) - # Advance by the uncropped count so the next request starts at the correct - # offset even if this row hit `table_width`. - source_offset += num_blocks_for_row - # Keep the graph-captured device address stable and transfer only the compact - # rectangle prepared in pinned host memory. - block_tables[:update_rows, :update_width].copy_( - host_block_tables[:update_rows, :update_width], - non_blocking=True) - wrappers.decode_block_table_active_rows = rows - wrappers.decode_block_table_active_width = active_width - kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) - if kv_lens_buf is not None: - decode_kv_lens = _to_int32_tensor( - kv_lens_host[self.num_contexts:self.num_contexts + - self.num_generations]) - kv_lens_buf[:self.num_generations].copy_(decode_kv_lens, - non_blocking=True) - if self.num_generations < batch_size: - kv_lens_buf[self.num_generations:batch_size].zero_() + When ``block_tables`` is not passed to + ``BatchDecodeWithPagedKVCacheWrapper.plan()``, flashinfer rebuilds + it with a per-request loop whose slice bounds are GPU scalars — + one cudaStreamSynchronize + one scalar D2H read per generation + request per plan. Instead, build the ``[num_gens, max_n]`` table + here with one vectorized pass over the host mirror of the active + pool's flat page indices (no GPU reads) and push it with a single + async H2D into a persistent device buffer held by ``wrappers``. + Under CUDA-graph metadata the buffer is allocated once at full + capacity width and never moves, so captured decode kernels keep + reading valid memory while prepare() refreshes the contents in + place; the eager path re-plans every step and may grow its buffer + geometrically. - # Refresh captured FA2 schedules only after all page metadata updates. - # Under CUDA graphs every non-trtllm-gen decode wrapper carries - # fa2_plan_num_blocks, so this is what keeps replayed plans current; - # the deferral below only affects wrappers without a recorded tuple. - # Defer ordinary multi-wrapper plans to forward_impl; single-wrapper - # models still plan eagerly because forward_impl cannot plan during - # graph capture. - active_wrappers = [ - pp for pp in self._plan_params_to_wrappers - if pp.attention_mask_data is None - ] - defer_plan = len(active_wrappers) > 1 - if not (self._is_separate_kv_draft_view and self.is_cuda_graph): - self._refresh_fa2_cuda_graph_plans() - self._clean_cached_plans(defer_plan=defer_plan) + Returns None when the host data (or memory for the buffer) is + unavailable; the caller then falls back to flashinfer's own + rebuild — itself sync-free now that the plan indptr is a host + tensor. + """ + if plan_params.attention_mask_data is not None: + # Masked plans are flushed every step, taking their wrappers + # (and any buffers on them) along; don't churn per-step + # block-table allocations for them. + return None + num_gens = self.num_generations + if num_gens == 0: + return None + host_paged_kv_indices = self._host_pool_indices.get( + plan_params.kv_pool_id, self._host_paged_kv_indices) + if host_paged_kv_indices is None: + return None + gen_num_blocks = np.asarray(self.num_blocks[self.num_contexts:], + dtype=np.int64) + max_n = int(gen_num_blocks.max()) + if max_n > self._max_num_blocks_per_seq: + # A request can never reference more than max_blocks_per_seq; + # defensive guard for inconsistent metadata. + return None + block_tables = wrappers.decode_block_tables + if (self.is_cuda_graph and block_tables is not None + and block_tables.size(1) < max_n): + # Never reallocate under CUDA graphs: captured decode kernels + # hold the buffer address. + return None + if block_tables is None or block_tables.size(1) < max_n: + if self.is_cuda_graph: + # Allocated once at capture warmup; full capacity width so + # replays never need a wider table. + width = self._max_num_blocks_per_seq + else: + # Eager path replans (and re-reads the table) every step, + # so the buffer may grow geometrically as sequences do. + width = min(max(64, 1 << (max_n - 1).bit_length()), + self._max_num_blocks_per_seq) + try: + block_tables = torch.zeros((self.max_num_requests, width), + dtype=torch.int32, + device='cuda') + except torch.OutOfMemoryError: + # E.g. the KV-estimation warmup forwards run with device + # memory deliberately exhausted; fall back to flashinfer's + # rebuild rather than failing the forward. + return None + wrappers.decode_block_tables = block_tables + host_block_tables = wrappers.host_decode_block_tables + if host_block_tables is None or host_block_tables.size(1) < max_n: + host_width = min(max(64, 1 << (max_n - 1).bit_length()), + block_tables.size(1)) + host_block_tables = torch.zeros((self.max_num_requests, host_width), + dtype=torch.int32, + pin_memory=prefer_pinned()) + wrappers.host_decode_block_tables = host_block_tables + start = self.num_context_blocks + decode_flat = host_paged_kv_indices.numpy()[start:start + + int(gen_num_blocks.sum())] + table = host_block_tables.numpy()[:num_gens, :max_n] + table[:] = 0 + table[np.arange(max_n)[None, :] < gen_num_blocks[:, None]] = \ + decode_flat + # Rewriting the host buffer is safe: _plan_with_params synchronizes + # the stream before planning, so the previous plan's H2D has + # completed. + block_tables[:num_gens, :max_n].copy_( + host_block_tables[:num_gens, :max_n], non_blocking=True) + # Seed the extent used by `prepare()` to clear entries if the first graph replay has fewer + # requests or a narrower block table. + wrappers.decode_block_table_active_rows = num_gens + wrappers.decode_block_table_active_width = max_n + return block_tables[:num_gens] - if (not self._is_shared_kv_draft_view - and not self._is_separate_kv_draft_view - and self._draft_metadata is not None): - self._draft_metadata._sync_draft_view(self) + def _clean_cached_plans(self, *, defer_plan: bool): + for plan_params in list(self._plan_params_to_wrappers.keys()): + # Generally, plan_params with non-trivial attention masking are relevant only the + # corresponding forward pass. So, flush them out here as they won't be relevant for + # subsequent forward calls. + if plan_params.attention_mask_data is None and plan_params.multi_item_params is None: + wrappers = self._plan_params_to_wrappers[plan_params] + if wrappers.fa2_plan_num_blocks is not None: + continue + wrappers.is_planned = False + if not defer_plan: + self._plan_with_params(plan_params) + else: + del self._plan_params_to_wrappers[plan_params] - if self.cross is not None and self.cross is not self: - self.cross.prepare() + def _refresh_fa2_cuda_graph_plans(self) -> None: + """Refresh captured FA2 schedules after page metadata is finalized.""" + num_blocks = tuple(self.num_blocks[self.num_contexts:]) + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + if (plan_params.attention_mask_data is not None + or plan_params.multi_item_params is not None + or wrappers.fa2_plan_num_blocks is None): + continue + if not num_blocks: + wrappers.fa2_plan_num_blocks = None + elif wrappers.fa2_plan_num_blocks != num_blocks: + # Graph replay does not re-enter forward_impl. Each wrapper + # owns its persistent integer plan workspace, while the shared + # float workspace is run scratch. + wrappers.is_planned = False + self._plan_with_params(plan_params) def plan(self, num_heads: int, @@ -2346,6 +2365,15 @@ def decode_plan(): return plan_params +class FlashInferAttentionMetadata(AttentionMetadata): + core_metadata_type: ClassVar[ + Type[AttentionCoreMetadata]] = FlashInferCoreMetadata + + # Report budget for the page-table check. A class attribute, because it + # bounds a run's log rather than a batch's. + _page_table_check_budget = _PAGE_TABLE_CHECK_REPORTS + + class FlashInferAttention(AttentionBackend[FlashInferAttentionMetadata]): Metadata = FlashInferAttentionMetadata @@ -2879,6 +2907,9 @@ def forward(self, forward_args: Optional[AttentionForwardArgs] = None, **kwargs) -> torch.Tensor: forward_args = merge_attention_forward_args(forward_args, kwargs) + if isinstance(metadata, AttentionMetadata): + # No forward on lengths changed in place without a refresh. + metadata.check_lengths_current() attention_mask_data = forward_args.attention_mask_data latent_cache = forward_args.latent_cache diff --git a/tensorrt_llm/_torch/attention/backends/fmha/cute_dsl_mla.py b/tensorrt_llm/_torch/attention/backends/fmha/cute_dsl_mla.py index 67bbf92df01f..5ed644e795cc 100644 --- a/tensorrt_llm/_torch/attention/backends/fmha/cute_dsl_mla.py +++ b/tensorrt_llm/_torch/attention/backends/fmha/cute_dsl_mla.py @@ -810,13 +810,10 @@ def _decode_kv_lengths( if lengths.data_ptr() % _KERNEL_INPUT_ALIGN_BYTES == 0: return lengths num_lengths = lengths.numel() - staging = getattr(meta, "_cute_dsl_mla_kv_lens_staging", None) - if staging is None or staging.numel() < num_lengths or staging.device != lengths.device: - if lengths.is_cuda and torch.cuda.is_current_stream_capturing(): - return cls._aligned_kernel_input(lengths) - capacity = max(int(getattr(meta, "max_num_requests", 0) or 0), num_lengths) - staging = torch.empty(capacity, dtype=torch.int32, device=lengths.device) - meta._cute_dsl_mla_kv_lens_staging = staging + staging_of = getattr(meta, "aligned_kv_lens_staging", None) + staging = staging_of(num_lengths, lengths.device) if staging_of is not None else None + if staging is None: + return cls._aligned_kernel_input(lengths) staged = staging[:num_lengths].view(lengths.shape) staged.copy_(lengths) return staged @@ -861,18 +858,11 @@ def _split_kv_plan( if not 0 < batch_size <= _SPLIT_PLAN_MAX_REQUESTS: return None - plan = getattr(meta, "_cute_dsl_mla_split_plan", None) - capacity = max(int(getattr(meta, "max_num_requests", 0) or 0), batch_size) - if plan is None or plan.numel() < batch_size or plan.device != cache_seqs.device: - if torch.cuda.is_current_stream_capturing(): - return None - plan = torch.empty(capacity, dtype=torch.int32, device=cache_seqs.device) - meta._cute_dsl_mla_split_plan = plan runner = CuteDSLNVMlaDecodeBlackwellRunner rows_per_request = runner.get_num_q_tiles(num_heads, seq_len_q) wave_clusters = max(runner._get_max_active_blocks() // runner._CLUSTER_SHAPE_MNK[0], 1) - key = (batch_size, rows_per_request, wave_clusters, cache_seqs.data_ptr(), plan.data_ptr()) - if getattr(meta, "_cute_dsl_mla_split_plan_key", None) != key: + + def launch(plan: torch.Tensor) -> None: launch_mla_split_kv_plan( cache_seqs, plan, @@ -880,8 +870,13 @@ def _split_kv_plan( rows_per_request=rows_per_request, wave_clusters=wave_clusters, ) - meta._cute_dsl_mla_split_plan_key = key - return plan[:batch_size] + + return meta.cute_dsl_mla_split_plan( + batch_size, + cache_seqs.device, + (batch_size, rows_per_request, wave_clusters, cache_seqs.data_ptr()), + launch, + ) def run_mla_generation( self, diff --git a/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py b/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py index 353a8384d00b..1c882acda600 100644 --- a/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py +++ b/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py @@ -38,6 +38,7 @@ ) from tensorrt_llm.bindings import DataType +from ..fp4_mla.state import find_fp4_mla from .interface import FmhaPhase from .phased import FmhaParams, PhasedFmha @@ -109,7 +110,7 @@ def _is_supported( ) ): raise NotImplementedError("FP4 MLA does not support sparse attention.") - state = metadata.fp4_mla_state + state = find_fp4_mla(metadata) if state is None or state.hp_pool is None: raise RuntimeError("FP4 MLA requires prepared high-precision KV state.") attention_input_type = forward_args.attention_input_type @@ -195,7 +196,7 @@ def run_mla_context(self, params: FmhaParams) -> None: raise RuntimeError("FP4 MLA context requires an output buffer.") if forward_args.latent_cache is None: raise RuntimeError("FP4 MLA context requires latent_cache.") - if metadata.fp4_mla_state.positions is None: + if find_fp4_mla(metadata).positions is None: raise RuntimeError("FP4 MLA context requires token positions.") if metadata.num_contexts <= 0: raise RuntimeError("FP4 MLA context requires context requests.") @@ -342,6 +343,7 @@ def forward_context_partition( def run_mla_generation(self, params: FmhaParams) -> None: attn = params.attn metadata = params.meta + fp4_mla_state = find_fp4_mla(metadata) forward_args = params.fwd q = params.query_input output = params.output @@ -359,18 +361,18 @@ def run_mla_generation(self, params: FmhaParams) -> None: qk_rope_head_dim = attn.qk_rope_head_dim or 0 fused_head_dim = kv_lora_rank + qk_rope_head_dim - if not bool(getattr(metadata.fp4_mla_state, "generation_cache_scattered", False)): + if not bool(getattr(fp4_mla_state, "generation_cache_scattered", False)): raise RuntimeError( "FP4 MLA generation requires fused RoPE/Q quantization and " "cache/HP-pool update before attention." ) - metadata.fp4_mla_state.generation_cache_scattered = False + fp4_mla_state.generation_cache_scattered = False query = q.view(q.shape[0], attn.num_heads, fused_head_dim) output_view = output.view(q.shape[0], attn.num_heads, kv_lora_rank) sm_scale = 1.0 / (attn.q_scaling * ((attn.qk_nope_head_dim or 0) + qk_rope_head_dim) ** 0.5) - prequantized_q = getattr(metadata.fp4_mla_state, "prequantized_q", None) - prequantized_q_sf = getattr(metadata.fp4_mla_state, "prequantized_q_sf", None) - q_batch_capacity = getattr(metadata.fp4_mla_state, "q_batch_capacity", None) + prequantized_q = getattr(fp4_mla_state, "prequantized_q", None) + prequantized_q_sf = getattr(fp4_mla_state, "prequantized_q_sf", None) + q_batch_capacity = getattr(fp4_mla_state, "q_batch_capacity", None) try: run_fp4_mla_attention_decode( metadata, @@ -387,10 +389,10 @@ def run_mla_generation(self, params: FmhaParams) -> None: softmax_stats_tensor=params.fwd.softmax_stats_tensor, ) finally: - metadata.fp4_mla_state.prequantized_q = None - metadata.fp4_mla_state.prequantized_q_sf = None - metadata.fp4_mla_state.q_batch_capacity = None - metadata.fp4_mla_state.prequantized_q_rows_per_seq = None + fp4_mla_state.prequantized_q = None + fp4_mla_state.prequantized_q_sf = None + fp4_mla_state.q_batch_capacity = None + fp4_mla_state.prequantized_q_rows_per_seq = None __all__ = ["Fp4MlaFmha"] diff --git a/tensorrt_llm/_torch/attention/backends/fmha/msa_prefill.py b/tensorrt_llm/_torch/attention/backends/fmha/msa_prefill.py index b31151545d19..703ee423def9 100644 --- a/tensorrt_llm/_torch/attention/backends/fmha/msa_prefill.py +++ b/tensorrt_llm/_torch/attention/backends/fmha/msa_prefill.py @@ -246,9 +246,9 @@ def run_msa_nvfp4_sparse_gqa( # A short pure-prefill step hands every layer the same identity selection (staged once per step by the # backend), so its k2q CSR worklist is layer-invariant: build it for the first layer and reuse it. - identity = metadata._msa_prefill_identity + identity = metadata.msa_prefill_identity csr_key = (batch, int(kv_block_indexes.shape[0])) - cached = metadata._msa_identity_csr + cached = metadata.msa_identity_csr if ( identity is not None and cached is not None @@ -271,7 +271,7 @@ def run_msa_nvfp4_sparse_gqa( return_schedule=True, ) if identity is not None: - metadata._msa_identity_csr = (identity, csr_key, (k2q_row_ptr, k2q_q_indices, schedule)) + metadata.cache_identity_csr((identity, csr_key, (k2q_row_ptr, k2q_q_indices, schedule))) result = sparse.sparse_atten_nvfp4_kv_func( q, k_paged.view(torch.uint8), diff --git a/tensorrt_llm/_torch/attention/backends/fmha/phased.py b/tensorrt_llm/_torch/attention/backends/fmha/phased.py index 22ed69809be3..8a972cc1353d 100644 --- a/tensorrt_llm/_torch/attention/backends/fmha/phased.py +++ b/tensorrt_llm/_torch/attention/backends/fmha/phased.py @@ -27,7 +27,6 @@ ) from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2, Role from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager -from tensorrt_llm._utils import get_sm_version from .interface import Fmha @@ -105,22 +104,7 @@ def get_spec_decoding_position_offsets( """Return the active int32 offsets as a zero-copy [requests, query width] view.""" if not (metadata.is_spec_decoding_enabled and metadata.use_spec_decoding): return None - offsets = metadata.spec_decoding_position_offsets - if offsets is not None and offsets.dim() == 1: - if not metadata.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()): - # Hopper offsets use the current compact query width, which can be - # smaller than the persistent buffer's capacity. - query_len = metadata.spec_decoding_query_len - if query_len <= 0: - raise ValueError( - "1-D speculative position offsets require a positive query length." - ) - offsets = offsets[: metadata.max_num_requests * query_len].view( - metadata.max_num_requests, query_len - ) - else: - offsets = offsets.view(metadata.max_num_requests, -1) - return offsets + return metadata.position_offsets_for_cpp def _leading_rows(t: torch.Tensor, n: int) -> torch.Tensor: diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py index fa8879296835..cc88ea8f3898 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py @@ -15,6 +15,7 @@ _validate_fp4_mla_kv_storage_shape, ) from .metadata import _materialize_fp4_mla_device_page_table_for_forward +from .state import find_fp4_mla def load_fp4_mla_chunked_kv_cache( @@ -30,6 +31,7 @@ def load_fp4_mla_chunked_kv_cache( qk_rope_head_dim: int, ) -> tuple[torch.Tensor, torch.Tensor]: """Gather one cached-prefix partition from dense FP4 MLA V2 storage.""" + fp4_mla_state = find_fp4_mla(metadata) if out_dtype not in (torch.float16, torch.bfloat16, torch.float32): raise TypeError(f"FP4 MLA chunk gather does not support output dtype {out_dtype}.") if num_ctx_cached_tokens < 0: @@ -87,11 +89,11 @@ def load_fp4_mla_chunked_kv_cache( if num_ctx_cached_tokens == 0: return compressed_kv, k_pe - if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): + if not bool(getattr(fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA chunk gather requires fixed-stride device page metadata.") _materialize_fp4_mla_device_page_table_for_forward(metadata) - page_table_stride = int(metadata.fp4_mla_state.page_table_stride) - page_ids = metadata.fp4_mla_state._paged_kv_indices + page_table_stride = int(fp4_mla_state.page_table_stride) + page_ids = fp4_mla_state._paged_kv_indices if ( page_table_stride <= 0 or not isinstance(page_ids, torch.Tensor) diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py index 47c80e861731..c84646efc5be 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py @@ -56,6 +56,7 @@ _materialize_fp4_mla_device_page_table_for_forward, ) from .packed_plan import _fp4_mla_helix_plan_bounds, _fp4_mla_packed_plan_geometry +from .state import find_fp4_mla from .v_cache import ( _get_cutedsl_persistent_v_packed_cache, _get_fp4_mla_v_packed_pool_base, @@ -272,6 +273,7 @@ def _scatter_fp4_mla_kv_cache_2d_context( v_packed_base: Optional[torch.Tensor] = None, v_page_offset: int = 0, ) -> bool: + fp4_mla_state = find_fp4_mla(metadata) num_contexts = metadata.num_contexts if num_contexts > 0: prompt_lens_cpu = metadata.prompt_lens_cpu_runtime[:num_contexts] @@ -319,7 +321,7 @@ def _scatter_fp4_mla_kv_cache_2d_context( q_context_view = latent_cache q_head_blocks = 0 - hp_pool = getattr(metadata.fp4_mla_state, "hp_pool", None) + hp_pool = getattr(fp4_mla_state, "hp_pool", None) if not isinstance(hp_pool, torch.Tensor): raise TypeError("FP4 MLA high-precision KV pool must be a tensor.") if hp_pool.device != latent_cache.device: @@ -338,7 +340,7 @@ def _scatter_fp4_mla_kv_cache_2d_context( ) if hp_pool.stride(-1) != 1: raise ValueError("FP4 MLA high-precision KV pool must be contiguous in head_dim.") - hp_page_ids = metadata.fp4_mla_state.hp_page_indices + hp_page_ids = fp4_mla_state.hp_page_indices if not isinstance(hp_page_ids, torch.Tensor): raise RuntimeError("FP4 MLA context cache update requires HP page metadata.") store_hp_tail = num_contexts > 0 @@ -372,13 +374,13 @@ def _scatter_fp4_mla_kv_cache_2d_context( rotary_cos_sin_ptr, hp_pool, hp_page_ids, - metadata.fp4_mla_state.batch_indices, - metadata.fp4_mla_state.positions, - metadata.fp4_mla_state.paged_kv_indices, - metadata.fp4_mla_state.paged_kv_indptr, - metadata.fp4_mla_state.paged_kv_indices.shape[0], - metadata.fp4_mla_state.paged_kv_indptr.shape[0], - metadata.fp4_mla_state.batch_indices.shape[0], + fp4_mla_state.batch_indices, + fp4_mla_state.positions, + fp4_mla_state.paged_kv_indices, + fp4_mla_state.paged_kv_indptr, + fp4_mla_state.paged_kv_indices.shape[0], + fp4_mla_state.paged_kv_indptr.shape[0], + fp4_mla_state.batch_indices.shape[0], num_pages, num_layers, num_contexts, @@ -454,6 +456,7 @@ def _scatter_fp4_mla_kv_cache_2d_generation( helix_position_offsets: Optional[torch.Tensor], helix_is_inactive_rank: Optional[torch.Tensor], ) -> Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: + fp4_mla_state = find_fp4_mla(metadata) num_contexts = metadata.num_contexts num_seqs = metadata.num_seqs num_gen = num_seqs - num_contexts @@ -494,7 +497,7 @@ def _scatter_fp4_mla_kv_cache_2d_generation( ) _materialize_fp4_mla_device_page_table_for_forward(metadata, kv_lens_gen) - pool = getattr(metadata.fp4_mla_state, "hp_pool", None) + pool = getattr(fp4_mla_state, "hp_pool", None) if pool is None: raise RuntimeError("FP4 MLA 2D generation scatter requires the HP KV pool.") try: @@ -688,7 +691,7 @@ def launch_generation_update( helix_is_inactive_rank, page_ids, hp_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + find_fp4_mla(metadata).paged_kv_indptr_decode, page_ids_len, hp_page_ids.numel(), indptr_len, @@ -749,7 +752,7 @@ def launch_generation_update( launch_generation_update( launch_grid, page_ids_len=page_ids.shape[0], - indptr_len=metadata.fp4_mla_state.paged_kv_indptr_decode.shape[0], + indptr_len=fp4_mla_state.paged_kv_indptr_decode.shape[0], max_gen_tiles_variant=max(max_gen_tiles, 1), q_prefix_block_dim_variant=q_prefix_block_dim, q_prefix_blocks_variant=q_prefix_blocks, @@ -810,12 +813,13 @@ def scatter_fp4_mla_kv_cache( backend-ready residual FP4 Q. The return value reports whether the current phase updated the HP pool. """ + fp4_mla_state = find_fp4_mla(metadata) if phase == "generation": - metadata.fp4_mla_state.generation_cache_scattered = False - metadata.fp4_mla_state.prequantized_q = None - metadata.fp4_mla_state.prequantized_q_sf = None - metadata.fp4_mla_state.q_batch_capacity = None - metadata.fp4_mla_state.prequantized_q_rows_per_seq = None + fp4_mla_state.generation_cache_scattered = False + fp4_mla_state.prequantized_q = None + fp4_mla_state.prequantized_q_sf = None + fp4_mla_state.q_batch_capacity = None + fp4_mla_state.prequantized_q_rows_per_seq = None if latent_cache.numel() == 0: raise ValueError("FP4 MLA cache scatter requires at least one latent token.") if q_context is not None and not latent_cache.is_contiguous(): @@ -831,8 +835,8 @@ def scatter_fp4_mla_kv_cache( raise ValueError( f"FP4 MLA KV head_dim must be divisible by {FP4_BLOCK_SIZE}, got {head_dim}." ) - indices_len = metadata.fp4_mla_state.batch_indices.shape[0] - positions_len = metadata.fp4_mla_state.positions.shape[0] + indices_len = fp4_mla_state.batch_indices.shape[0] + positions_len = fp4_mla_state.positions.shape[0] if token_offset + num_tokens > indices_len or token_offset + num_tokens > positions_len: raise RuntimeError( f"FP4 MLA scatter would read batch_indices[{token_offset}:" @@ -864,7 +868,7 @@ def scatter_fp4_mla_kv_cache( context_only_cache = getattr(kv_cache_manager, "fp4_mla_context_only_cache", False) is True if context_only_cache and phase != "context": raise RuntimeError("FP4 MLA context-only cache cannot execute generation.") - if not context_only_cache and getattr(metadata.fp4_mla_state, "v_scale_pool", None) is None: + if not context_only_cache and getattr(fp4_mla_state, "v_scale_pool", None) is None: raise RuntimeError("FP4 MLA scatter requires the auxiliary V scale pool.") if metadata.page_size % FP4_BLOCK_SIZE != 0: raise ValueError( @@ -934,10 +938,10 @@ def scatter_fp4_mla_kv_cache( q_quant_input.device, rows_per_seq=q_rows_per_seq, ) - metadata.fp4_mla_state.prequantized_q = q_fp4_out - metadata.fp4_mla_state.prequantized_q_sf = q_sf_out - metadata.fp4_mla_state.q_batch_capacity = q_batch_capacity - metadata.fp4_mla_state.prequantized_q_rows_per_seq = q_rows_per_seq + fp4_mla_state.prequantized_q = q_fp4_out + fp4_mla_state.prequantized_q_sf = q_sf_out + fp4_mla_state.q_batch_capacity = q_batch_capacity + fp4_mla_state.prequantized_q_rows_per_seq = q_rows_per_seq hp_pool_updated = True cutedsl_backend = _fp4_mla_attention_backend() == _FP4_MLA_CUTEDSL_BACKEND fused_v_transpose = cutedsl_backend and _fp4_mla_cutedsl_fused_v_transpose_enabled() @@ -1031,7 +1035,7 @@ def scatter_fp4_mla_kv_cache( v_packed_base=v_packed_base, v_page_offset=v_page_offset, ) - v_pack_page_ids = metadata.fp4_mla_state.paged_kv_indices + v_pack_page_ids = fp4_mla_state.paged_kv_indices else: generation_state = _scatter_fp4_mla_kv_cache_2d_generation( metadata, @@ -1078,20 +1082,18 @@ def scatter_fp4_mla_kv_cache( ): if phase == "context": num_contexts = metadata.num_contexts - cutedsl_v_pack_page_ids = metadata.fp4_mla_state.paged_kv_indices[ - : metadata.fp4_mla_state.num_context_blocks + cutedsl_v_pack_page_ids = fp4_mla_state.paged_kv_indices[ + : fp4_mla_state.num_context_blocks ] - cutedsl_repack_page_indptr = metadata.fp4_mla_state.paged_kv_indptr[: num_contexts + 1] + cutedsl_repack_page_indptr = fp4_mla_state.paged_kv_indptr[: num_contexts + 1] cutedsl_repack_kv_lens = metadata.kv_lens_cuda_runtime[:num_contexts] cutedsl_repack_generation_lens = metadata.prompt_lens_cuda_runtime[:num_contexts] - cutedsl_repack_max_touched_pages = int( - metadata.fp4_mla_state.context_repack_max_touched_pages - ) + cutedsl_repack_max_touched_pages = int(fp4_mla_state.context_repack_max_touched_pages) elif generation_state is not None: kv_lens_gen, gen_lens_gen, generation_page_ids = generation_state num_gen = kv_lens_gen.numel() cutedsl_v_pack_page_ids = generation_page_ids - cutedsl_repack_page_indptr = metadata.fp4_mla_state.paged_kv_indptr_decode + cutedsl_repack_page_indptr = fp4_mla_state.paged_kv_indptr_decode cutedsl_repack_kv_lens = kv_lens_gen cutedsl_repack_generation_lens = gen_lens_gen cutedsl_repack_max_touched_pages = _ceil_div( @@ -1127,5 +1129,5 @@ def scatter_fp4_mla_kv_cache( num_valid_pages=v_pack_num_valid, ) if phase == "generation": - metadata.fp4_mla_state.generation_cache_scattered = hp_pool_updated + fp4_mla_state.generation_cache_scattered = hp_pool_updated return hp_pool_updated diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py index 372a442026fa..dc820ecc57b1 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py @@ -55,6 +55,7 @@ _launch_fp4_mla_packed_plan, _PackedPlanGeometry, ) +from .state import find_fp4_mla from .v_cache import ( _get_cutedsl_persistent_v_packed_cache, _get_fp4_mla_v_packed_pool_base, @@ -166,6 +167,7 @@ def _run_triton_attention_decode( ``fp4_mla_triton.py``. Threads through the constexpr assume flags, TMA descriptors, occupancy/num-warps launch meta, and pipelined PV loop. """ + fp4_mla_state = find_fp4_mla(metadata) from .fp4_mla_triton import ( _fp4_mla_attention_group_reduce_stats_kernel as _attn_group_reduce_stats_kernel, ) @@ -312,7 +314,7 @@ def _tma_alloc(size: int, alignment: int, stream): global_scale, q_global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -377,8 +379,8 @@ def _tma_alloc(size: int, alignment: int, stream): # reduce during capture so we never allocate mid-capture. The single-level # reduce is numerically identical (it just launches fewer CTAs). if num_reduce_groups > 1 and torch.cuda.is_current_stream_capturing(): - gmax = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_group_max_buf") - gsum = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_group_sum_buf") + gmax = fp4_mla_state.workspaces.get("_fp4_mla_attention_group_max_buf") + gsum = fp4_mla_state.workspaces.get("_fp4_mla_attention_group_sum_buf") groups_ready = ( gmax is not None and gsum is not None @@ -458,7 +460,7 @@ def _tma_alloc(size: int, alignment: int, stream): max_scores, denom, page_max, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], max_scores.stride(0), @@ -528,7 +530,7 @@ def _tma_alloc(size: int, alignment: int, stream): # Fall back to the unsplit PV (numerically identical) during capture unless a # warmup forward already sized that workspace, so capture never allocates. if page_split > 1 and torch.cuda.is_current_stream_capturing(): - pbuf = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_pv_partial_buf") + pbuf = fp4_mla_state.workspaces.get("_fp4_mla_attention_pv_partial_buf") partial_ready = ( pbuf is not None and pbuf.shape[0] >= num_queries @@ -558,7 +560,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -607,7 +609,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -681,7 +683,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -722,7 +724,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -854,7 +856,7 @@ def _run_cutedsl_packed_decode( plan = plan_geometry.plan # The first FP4 MLA kernel of the step wrote the plan fused with the # generation lengths; a stale key means this call plans on other bounds. - if metadata.fp4_mla_state.packed_plan_key != plan_geometry.key: + if find_fp4_mla(metadata).packed_plan_key != plan_geometry.key: _launch_fp4_mla_packed_plan(plan_geometry, kv_lens, metadata) partial = _ensure_workspace_tensor( metadata, @@ -937,6 +939,7 @@ def run_fp4_mla_attention_decode( KV workspace is materialized on this path. Callers must supply the packed Q and scales produced by the fused generation cache update. """ + fp4_mla_state = find_fp4_mla(metadata) head_dim = kv_lora_rank + qk_rope_head_dim if qk_rope_head_dim != FP4_MLA_K_RESIDUAL_DIM: raise ValueError( @@ -1010,7 +1013,7 @@ def run_fp4_mla_attention_decode( "same-device int32 tensor covering every query token." ) helix_kv_bounds = helix_kv_bounds[:num_queries] - if getattr(metadata.fp4_mla_state, "v_scale_pool", None) is None: + if getattr(fp4_mla_state, "v_scale_pool", None) is None: raise RuntimeError( "FP4 MLA attention decode requires the auxiliary V scale pool to be allocated." ) @@ -1027,9 +1030,7 @@ def run_fp4_mla_attention_decode( capacity = int(q_batch_capacity) # The fused generation update records the Q layout it produced; metadata # that never went through it (contract tests) carries the legacy layout. - recorded_rows_per_seq = getattr( - getattr(metadata, "fp4_mla_state", None), "prequantized_q_rows_per_seq", None - ) + recorded_rows_per_seq = getattr(fp4_mla_state, "prequantized_q_rows_per_seq", None) packed_rows_per_seq = int(recorded_rows_per_seq or 0) if recorded_rows_per_seq is not None and packed_rows_per_seq != _fp4_mla_packed_rows_per_seq( num_heads, query_len_per_seq @@ -1089,7 +1090,7 @@ def run_fp4_mla_attention_decode( ) sf_cache = sf_cache.view(torch.float8_e4m3fn) - v_sf_pool = metadata.fp4_mla_state.v_scale_pool + v_sf_pool = fp4_mla_state.v_scale_pool v_sf = get_fp4_mla_v_scale_pool_view(metadata, v_head_dim=kv_lora_rank)[local_layer].view( torch.float8_e4m3fn ) @@ -1280,7 +1281,7 @@ def run_fp4_mla_attention_decode( global_scale, q_global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, output, num_heads=num_heads, @@ -1311,7 +1312,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, output, splits=select_split_count(max_pages * metadata.page_size), @@ -1363,7 +1364,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, kernel_output, splits=splits, @@ -1384,7 +1385,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, kernel_output, max_kv_len=max_pages * metadata.page_size, diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py index b5fbcfe33fb2..8b2dbf01fa36 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py @@ -25,6 +25,7 @@ from tensorrt_llm.quantization.mode import QuantMode from .fp4_mla_kernels import _fp8_mla_context_block_table_kernel +from .state import Fp4MlaState, find_fp4_mla if TYPE_CHECKING: from tensorrt_llm._torch.attention.backends.trtllm import ( @@ -340,25 +341,22 @@ def _build_fp8_mla_context_metadata( kv_lens_cuda = meta.prompt_lens_cuda_runtime[: meta.num_contexts] if kv_lens_cpu is None: kv_lens_cpu = meta.prompt_lens_cpu_runtime[: meta.num_contexts] - fp8_meta = copy.copy(meta) - fp8_meta.fp4_mla_state = None - fp8_meta.kv_cache_manager = scratch.cache_manager_view - fp8_meta.kv_cache_block_offsets = scratch.block_offsets - fp8_meta.block_ids_per_seq = scratch.block_ids_per_seq - fp8_meta.prompt_lens_cuda_runtime = meta.prompt_lens_cuda_runtime[: meta.num_contexts] - fp8_meta.prompt_lens_cpu_runtime = meta.prompt_lens_cpu_runtime[: meta.num_contexts] - fp8_meta.host_request_types_runtime = meta.host_request_types_runtime[: meta.num_contexts] # The disposable cache represents only this context invocation. Expose # exact context-only lengths so cached prefixes, trailing generation - # metadata, and stale totals cannot extend FP8 K/V quantization. - fp8_meta.kv_lens_cuda_runtime = kv_lens_cuda - fp8_meta.kv_lens_runtime = kv_lens_cpu - fp8_meta.host_total_kv_lens = scratch.host_total_kv_lens - # Scratch lengths intentionally start from zero. Preserve the actual - # absolute positions for Q/K RoPE through the native kernel's explicit - # per-token position-offset input. - fp8_meta.helix_position_offsets = meta.fp4_mla_state.positions[: meta.num_ctx_tokens] - return fp8_meta + # metadata, and stale totals cannot extend FP8 K/V quantization. Scratch + # lengths intentionally start from zero; the actual absolute positions + # reach Q/K RoPE through the native kernel's explicit per-token + # position-offset input. + return meta.context_scratch_view( + kv_cache_manager=scratch.cache_manager_view, + kv_cache_block_offsets=scratch.block_offsets, + block_ids_per_seq=scratch.block_ids_per_seq, + host_total_kv_lens=scratch.host_total_kv_lens, + kv_lens_cuda=kv_lens_cuda, + kv_lens=kv_lens_cpu, + position_offsets=find_fp4_mla(meta).positions[: meta.num_ctx_tokens], + without=(Fp4MlaState,), + ) def _get_fp8_mla_context_metadata( @@ -366,11 +364,12 @@ def _get_fp8_mla_context_metadata( scratch: _Fp8MlaContextScratch, ) -> "TrtllmAttentionMetadata": """Prepare and reuse one direct metadata view for all layers in a step.""" - state = meta.fp4_mla_state.fp8_context_state + fp4_mla_state = find_fp4_mla(meta) + state = fp4_mla_state._fp8_context_view if state is not None and state[0] is scratch: return state[1] scratch.prepare(meta) fp8_meta = _build_fp8_mla_context_metadata(meta, scratch) - meta.fp4_mla_state.fp8_context_state = (scratch, fp8_meta) + fp4_mla_state._fp8_context_view = (scratch, fp8_meta) return fp8_meta diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py index e4bdf6e2c739..d8ef69317a39 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py @@ -17,6 +17,7 @@ HP_BLOCK_SIZE, _ceil_div, ) +from .state import find_fp4_mla # FP4 MLA scale-layout helpers @@ -69,7 +70,7 @@ def get_fp4_mla_v_scale_pool_view( v_head_dim: int, ) -> torch.Tensor: """View the auxiliary MLA V-scale pool in Triton's block-scaled layout.""" - pool = getattr(metadata.fp4_mla_state, "v_scale_pool", None) + pool = getattr(find_fp4_mla(metadata), "v_scale_pool", None) if pool is None: raise RuntimeError("FP4 MLA V scale pool is not allocated.") @@ -100,7 +101,7 @@ def get_fp4_mla_v_scale_pool_view( def _get_fp4_mla_global_scale(metadata: Any, device: torch.device) -> torch.Tensor: - global_scale = getattr(metadata.fp4_mla_state, "kv_global_scale", None) + global_scale = getattr(find_fp4_mla(metadata), "kv_global_scale", None) if ( not isinstance(global_scale, torch.Tensor) or global_scale.device != device @@ -112,7 +113,7 @@ def _get_fp4_mla_global_scale(metadata: Any, device: torch.device) -> torch.Tens def _get_fp4_mla_q_global_scale(metadata: Any, device: torch.device) -> torch.Tensor: - global_scale = getattr(metadata.fp4_mla_state, "q_global_scale", None) + global_scale = getattr(find_fp4_mla(metadata), "q_global_scale", None) if ( not isinstance(global_scale, torch.Tensor) or global_scale.device != device @@ -235,7 +236,7 @@ def _ensure_workspace_tensor( dtype: torch.dtype, device: torch.device, ) -> torch.Tensor: - workspaces = metadata.fp4_mla_state.workspaces + workspaces = find_fp4_mla(metadata).workspaces tensor = workspaces.get(attr_name) needs_alloc = ( tensor is None diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py index c248860aace5..f4866395613b 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py @@ -10,6 +10,7 @@ from .config import _FP4_MLA_PAGE_TABLE_TILE_SIZE, FP4_MLA_TOKENS_PER_BLOCK, _ceil_div from .packed_plan import _launch_fp4_mla_packed_plan, _PackedPlanGeometry +from .state import find_fp4_mla @triton.jit @@ -490,10 +491,11 @@ def configure_fp4_mla_device_page_table( table on the GPU. The materialization kernel decodes V2 page indices and refreshes rows from the final device KV lengths before cache update. """ - metadata.fp4_mla_state.device_page_table = False - metadata.fp4_mla_state.device_page_table_valid = False - metadata.fp4_mla_state.page_table_stride = 0 - metadata.fp4_mla_state.context_repack_max_touched_pages = 1 + fp4_mla_state = find_fp4_mla(metadata) + fp4_mla_state.device_page_table = False + fp4_mla_state.device_page_table_valid = False + fp4_mla_state.page_table_stride = 0 + fp4_mla_state.context_repack_max_touched_pages = 1 kv_cache_manager = getattr(metadata, "kv_cache_manager", None) num_contexts = int(getattr(metadata, "num_contexts", 0)) @@ -503,9 +505,9 @@ def configure_fp4_mla_device_page_table( num_context_tokens = int(getattr(metadata, "num_ctx_tokens", 0)) num_generation_tokens = num_tokens - num_context_tokens block_offsets = getattr(metadata, "kv_cache_block_offsets", None) - page_ids = getattr(metadata.fp4_mla_state, "_paged_kv_indices", None) - paged_kv_indptr = getattr(metadata.fp4_mla_state, "_paged_kv_indptr", None) - paged_kv_indptr_decode = getattr(metadata.fp4_mla_state, "paged_kv_indptr_decode", None) + page_ids = getattr(fp4_mla_state, "_paged_kv_indices", None) + paged_kv_indptr = getattr(fp4_mla_state, "_paged_kv_indptr", None) + paged_kv_indptr_decode = getattr(fp4_mla_state, "paged_kv_indptr_decode", None) max_page_capacity = int(getattr(kv_cache_manager, "max_blocks_per_seq", 0) or 0) page_spec = _fp4_mla_page_table_spec(kv_cache_manager) page_index_scale = int(page_spec.cache_page_index_scale) @@ -537,7 +539,7 @@ def configure_fp4_mla_device_page_table( if not supported: return False - hp_page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) + hp_page_ids = getattr(fp4_mla_state, "hp_page_indices", None) max_pool_id = max(page_spec.cache_pool_id, page_spec.hp_pool_id) if ( not isinstance(hp_page_ids, torch.Tensor) @@ -546,10 +548,10 @@ def configure_fp4_mla_device_page_table( or block_offsets.shape[0] <= max_pool_id ): return False - metadata.fp4_mla_state.cache_pool_id = int(page_spec.cache_pool_id) - metadata.fp4_mla_state.cache_page_index_scale = int(page_spec.cache_page_index_scale) - metadata.fp4_mla_state.hp_pool_id = int(page_spec.hp_pool_id) - metadata.fp4_mla_state.hp_page_index_scale = int(page_spec.hp_page_index_scale) + fp4_mla_state.cache_pool_id = int(page_spec.cache_pool_id) + fp4_mla_state.cache_page_index_scale = int(page_spec.cache_page_index_scale) + fp4_mla_state.hp_pool_id = int(page_spec.hp_pool_id) + fp4_mla_state.hp_page_index_scale = int(page_spec.hp_page_index_scale) max_pages = max_page_capacity host_kv_lens_available = ( @@ -581,7 +583,7 @@ def configure_fp4_mla_device_page_table( max_context_len, FP4_MLA_TOKENS_PER_BLOCK, ) - metadata.fp4_mla_state.context_repack_max_touched_pages = min( + fp4_mla_state.context_repack_max_touched_pages = min( max_pages, triton.next_power_of_2(max(1, max_context_pages)), ) @@ -606,15 +608,15 @@ def configure_fp4_mla_device_page_table( ) if not buffers_cover_table: return False - if metadata.fp4_mla_state.hp_page_indices.numel() < required_page_ids: + if fp4_mla_state.hp_page_indices.numel() < required_page_ids: return False - metadata.fp4_mla_state.device_page_table = True - metadata.fp4_mla_state.num_sequences = num_sequences - metadata.fp4_mla_state.page_table_stride = max_pages - metadata.fp4_mla_state.num_blocks = None - metadata.fp4_mla_state.num_context_blocks = num_contexts * max_pages - metadata.fp4_mla_state.num_generation_blocks = num_generation_sequences * max_pages + fp4_mla_state.device_page_table = True + fp4_mla_state.num_sequences = num_sequences + fp4_mla_state.page_table_stride = max_pages + fp4_mla_state.num_blocks = None + fp4_mla_state.num_context_blocks = num_contexts * max_pages + fp4_mla_state.num_generation_blocks = num_generation_sequences * max_pages return True @@ -624,15 +626,16 @@ def materialize_fp4_mla_device_page_table( generation_kv_lens: Optional[torch.Tensor] = None, ) -> None: """Refresh the fixed-stride context and generation page table once per forward.""" - if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): + fp4_mla_state = find_fp4_mla(metadata) + if not bool(getattr(fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA requires fixed-stride device page metadata.") - if bool(getattr(metadata.fp4_mla_state, "device_page_table_valid", False)): + if bool(getattr(fp4_mla_state, "device_page_table_valid", False)): return num_contexts = int(metadata.num_contexts) num_sequences = int(metadata.num_seqs) num_generation_sequences = num_sequences - num_contexts - max_pages = int(metadata.fp4_mla_state.page_table_stride) + max_pages = int(fp4_mla_state.page_table_stride) if num_sequences <= 0 or max_pages <= 0: raise RuntimeError( "FP4 MLA device page metadata requires positive sequence and page capacities." @@ -664,15 +667,15 @@ def materialize_fp4_mla_device_page_table( f"{num_generation_sequences} entries." ) - cache_pool_id = int(getattr(metadata.fp4_mla_state, "cache_pool_id", 0)) + cache_pool_id = int(getattr(fp4_mla_state, "cache_pool_id", 0)) block_offsets = metadata.kv_cache_block_offsets[ cache_pool_id, :num_sequences, 0, :max_pages, ] - page_ids = metadata.fp4_mla_state._paged_kv_indices[: num_sequences * max_pages] - page_index_scale = int(metadata.fp4_mla_state.cache_page_index_scale) + page_ids = fp4_mla_state._paged_kv_indices[: num_sequences * max_pages] + page_index_scale = int(fp4_mla_state.cache_page_index_scale) if page_index_scale <= 0: raise RuntimeError("FP4 MLA device page metadata requires a positive page-index scale.") grid = ( @@ -681,8 +684,8 @@ def materialize_fp4_mla_device_page_table( ) _fp4_mla_materialize_page_table_kernel[grid]( page_ids, - metadata.fp4_mla_state._paged_kv_indptr, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state._paged_kv_indptr, + fp4_mla_state.paged_kv_indptr_decode, block_offsets, kv_lens, generation_kv_lens, @@ -695,11 +698,11 @@ def materialize_fp4_mla_device_page_table( PAGE_TILE_SIZE=_FP4_MLA_PAGE_TABLE_TILE_SIZE, num_warps=4, ) - hp_page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) + hp_page_ids = getattr(fp4_mla_state, "hp_page_indices", None) if not isinstance(hp_page_ids, torch.Tensor): raise RuntimeError("FP4 MLA requires an HP page-table output tensor.") - hp_pool_id = int(metadata.fp4_mla_state.hp_pool_id) - hp_page_index_scale = int(metadata.fp4_mla_state.hp_page_index_scale) + hp_pool_id = int(fp4_mla_state.hp_pool_id) + hp_page_index_scale = int(fp4_mla_state.hp_page_index_scale) hp_block_offsets = metadata.kv_cache_block_offsets[ hp_pool_id, :num_sequences, @@ -708,8 +711,8 @@ def materialize_fp4_mla_device_page_table( ] _fp4_mla_materialize_page_table_kernel[grid]( hp_page_ids, - metadata.fp4_mla_state._paged_kv_indptr, - metadata.fp4_mla_state.paged_kv_indptr_decode, + fp4_mla_state._paged_kv_indptr, + fp4_mla_state.paged_kv_indptr_decode, hp_block_offsets, kv_lens, generation_kv_lens, @@ -722,7 +725,7 @@ def materialize_fp4_mla_device_page_table( PAGE_TILE_SIZE=_FP4_MLA_PAGE_TABLE_TILE_SIZE, num_warps=4, ) - metadata.fp4_mla_state.device_page_table_valid = True + fp4_mla_state.device_page_table_valid = True def _host_int_list_during_forward(value: Any, start: int, end: int) -> Optional[list[int]]: @@ -739,7 +742,7 @@ def _fp4_mla_generation_num_blocks_device(metadata: Any) -> torch.Tensor: slots are never consumed. """ num_gen = metadata.num_seqs - metadata.num_contexts - return metadata.fp4_mla_state.paged_kv_indptr_decode[num_gen : num_gen + 1] + return find_fp4_mla(metadata).paged_kv_indptr_decode[num_gen : num_gen + 1] def _fp4_mla_uniform_generation_lengths( @@ -754,6 +757,7 @@ def _fp4_mla_uniform_generation_lengths( step's packed split plan (one CuTe DSL kernel instead of two launches), and a stale plan is rewritten even when the lengths are already current. """ + fp4_mla_state = find_fp4_mla(metadata) if num_gen <= 0 or num_gen_tokens % num_gen != 0: raise RuntimeError("FP4 MLA generation requires a non-empty uniform request batch.") if plan_geometry is not None and plan_geometry.num_requests != num_gen: @@ -763,8 +767,8 @@ def _fp4_mla_uniform_generation_lengths( num_seqs = metadata.num_seqs kv_lens_gen = metadata.kv_lens_cuda_runtime[num_contexts:num_seqs] prompt_lens_gen = metadata.prompt_lens_cuda_runtime[num_contexts:num_seqs] - corrected_kv_lens = getattr(metadata.fp4_mla_state, "generation_kv_lens", None) - generation_lens = getattr(metadata.fp4_mla_state, "generation_append_lens", None) + corrected_kv_lens = getattr(fp4_mla_state, "generation_kv_lens", None) + generation_lens = getattr(fp4_mla_state, "generation_append_lens", None) tensors = ( kv_lens_gen, prompt_lens_gen, @@ -782,18 +786,18 @@ def _fp4_mla_uniform_generation_lengths( record_for_capture = bool( getattr(metadata, "is_cuda_graph", False) and torch.cuda.is_current_stream_capturing() - and not getattr(metadata.fp4_mla_state, "generation_lengths_capture_recorded", False) + and not getattr(fp4_mla_state, "generation_lengths_capture_recorded", False) ) precomputed = ( not record_for_capture - and metadata.fp4_mla_state.generation_lengths_num_tokens == num_gen_tokens - and metadata.fp4_mla_state.generation_lengths_num_seqs == num_gen - and metadata.fp4_mla_state.generation_lengths_num_contexts == num_contexts + and fp4_mla_state.generation_lengths_num_tokens == num_gen_tokens + and fp4_mla_state.generation_lengths_num_seqs == num_gen + and fp4_mla_state.generation_lengths_num_contexts == num_contexts ) plan_pending = ( plan_geometry is not None and plan_geometry.key is not None - and metadata.fp4_mla_state.packed_plan_key != plan_geometry.key + and fp4_mla_state.packed_plan_key != plan_geometry.key ) if not precomputed: if plan_pending: @@ -817,11 +821,11 @@ def _fp4_mla_uniform_generation_lengths( num_gen_tokens=num_gen_tokens, num_gen=num_gen, ) - metadata.fp4_mla_state.generation_lengths_num_tokens = num_gen_tokens - metadata.fp4_mla_state.generation_lengths_num_seqs = num_gen - metadata.fp4_mla_state.generation_lengths_num_contexts = num_contexts + fp4_mla_state.generation_lengths_num_tokens = num_gen_tokens + fp4_mla_state.generation_lengths_num_seqs = num_gen + fp4_mla_state.generation_lengths_num_contexts = num_contexts if record_for_capture: - metadata.fp4_mla_state.generation_lengths_capture_recorded = True + fp4_mla_state.generation_lengths_capture_recorded = True elif plan_pending: _launch_fp4_mla_packed_plan(plan_geometry, corrected_kv_lens[:num_gen], metadata) return corrected_kv_lens[:num_gen], generation_lens[:num_gen] @@ -832,9 +836,10 @@ def _materialize_fp4_mla_device_page_table_for_forward( generation_kv_lens: Optional[torch.Tensor] = None, ) -> None: """Materialize all fixed-stride rows from final per-forward device lengths.""" - if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): + fp4_mla_state = find_fp4_mla(metadata) + if not bool(getattr(fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA cache update requires fixed-stride device page metadata.") - if bool(getattr(metadata.fp4_mla_state, "device_page_table_valid", False)): + if bool(getattr(fp4_mla_state, "device_page_table_valid", False)): return num_contexts = int(metadata.num_contexts) @@ -858,12 +863,13 @@ def _materialize_fp4_mla_device_page_table_for_forward( def _max_generation_pages(metadata: Any) -> int: + fp4_mla_state = find_fp4_mla(metadata) num_gen = metadata.num_seqs - metadata.num_contexts if num_gen <= 0: return 0 - if not getattr(metadata.fp4_mla_state, "device_page_table", False): + if not getattr(fp4_mla_state, "device_page_table", False): raise RuntimeError("FP4 MLA generation requires fixed-stride device page metadata.") - max_pages = int(metadata.fp4_mla_state.page_table_stride) + max_pages = int(fp4_mla_state.page_table_stride) if max_pages <= 0: raise RuntimeError("FP4 MLA device page-table stride must be positive.") return max_pages @@ -878,7 +884,7 @@ def _fp4_mla_generation_page_ids(metadata: Any, num_gen_seqs: int) -> torch.Tens f"{num_gen_seqs} != {expected_num_gen}." ) max_pages = _max_generation_pages(metadata) - page_ids = getattr(metadata.fp4_mla_state, "_paged_kv_indices", None) + page_ids = getattr(find_fp4_mla(metadata), "_paged_kv_indices", None) start = metadata.num_contexts * max_pages end = start + num_gen_seqs * max_pages if ( @@ -901,7 +907,7 @@ def _fp4_mla_generation_hp_page_ids(metadata: Any, num_gen_seqs: int) -> torch.T f"{num_gen_seqs} != {expected_num_gen}." ) max_pages = _max_generation_pages(metadata) - page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) + page_ids = getattr(find_fp4_mla(metadata), "hp_page_indices", None) start = metadata.num_contexts * max_pages end = start + num_gen_seqs * max_pages if ( @@ -929,12 +935,13 @@ def _host_int_list(value: Any, start: int, end: int) -> Optional[list[int]]: def _infer_assume_full_pages(metadata: Any, max_pages: int, page_size: int) -> bool: + fp4_mla_state = find_fp4_mla(metadata) if getattr(metadata, "is_cuda_graph", False): return False start = metadata.num_contexts end = metadata.num_seqs - block_counts = _host_int_list(getattr(metadata.fp4_mla_state, "num_blocks", None), start, end) + block_counts = _host_int_list(getattr(fp4_mla_state, "num_blocks", None), start, end) if block_counts is not None and ( not block_counts or min(block_counts) != max_pages or max(block_counts) != max_pages ): @@ -950,12 +957,12 @@ def _infer_assume_full_pages(metadata: Any, max_pages: int, page_size: int) -> b tuple(block_counts) if block_counts is not None else None, kv_lens_cuda.data_ptr(), ) - cache = getattr(metadata.fp4_mla_state, "full_pages_cache", None) + cache = getattr(fp4_mla_state, "full_pages_cache", None) if cache is not None and cache[0] == cache_key: return bool(cache[1]) kv_lens = [int(item) for item in kv_lens_cuda[start:end].detach().cpu().tolist()] result = bool(kv_lens) and min(kv_lens) == max(kv_lens) == max_pages * page_size - setattr(metadata.fp4_mla_state, "full_pages_cache", (cache_key, result)) + setattr(fp4_mla_state, "full_pages_cache", (cache_key, result)) return result kv_cache_params = getattr(metadata, "kv_cache_params", None) diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py index 8c9817444a50..411cb9fc300c 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py @@ -18,6 +18,7 @@ _fp4_mla_packed_wave_plan_enabled, ) from .layout import _ensure_workspace_tensor +from .state import find_fp4_mla _SM_COUNT_CACHE: dict[int, int] = {} @@ -104,13 +105,14 @@ def _fp4_mla_packed_plan_geometry( generation lengths (``_fp4_mla_uniform_generation_lengths``). ``key`` is None when every request keeps a single task and no plan is needed. """ + fp4_mla_state = find_fp4_mla(metadata) rows_per_seq = _fp4_mla_packed_rows_per_seq(num_heads, query_len) if rows_per_seq == 0 or num_requests <= 0: return None - if not metadata.fp4_mla_state.device_page_table: + if not fp4_mla_state.device_page_table: return None # The fixed-stride generation page-table capacity (see _max_generation_pages). - max_pages = int(metadata.fp4_mla_state.page_table_stride) + max_pages = int(fp4_mla_state.page_table_stride) if max_pages <= 0: return None max_kv_len = max_pages * metadata.page_size @@ -164,7 +166,7 @@ def _fp4_mla_packed_plan_geometry( dtype=torch.int32, device=device, ) - kv_lens = metadata.fp4_mla_state.generation_kv_lens + kv_lens = fp4_mla_state.generation_kv_lens key = ( num_requests, splits, @@ -230,4 +232,4 @@ def _launch_fp4_mla_packed_plan( generation_lengths=generation_lengths, wave_plan=_fp4_mla_packed_wave_plan_enabled(), ) - metadata.fp4_mla_state.packed_plan_key = geometry.key + find_fp4_mla(metadata).packed_plan_key = geometry.key diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/state.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/state.py index 86de69be8607..c70969fc01bd 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/state.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/state.py @@ -4,22 +4,31 @@ from __future__ import annotations -from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from dataclasses import MISSING, dataclass, field, fields +from typing import TYPE_CHECKING, Any import torch +from ..extensions import MetadataExtension from .config import FP4_MLA_KV_GLOBAL_SCALE, FP4_MLA_Q_GLOBAL_SCALE, HP_BLOCK_SIZE -from .metadata import configure_fp4_mla_device_page_table, populate_fp4_mla_append_metadata if TYPE_CHECKING: - from ....memory_buffer_utils import Buffers from ..trtllm import TrtllmAttentionMetadata from .fp4_mla_context import _Fp8MlaContextScratch -@dataclass -class Fp4MlaState: +def find_fp4_mla(metadata: Any) -> Fp4MlaState | None: + """The FP4 MLA extension of ``metadata``, or None. + + Accepts the duck-typed metadata views the FP4 MLA helpers take, which may + carry no extensions at all. + """ + find_ext = getattr(metadata, "find_ext", None) + return find_ext(Fp4MlaState) if find_ext is not None else None + + +@dataclass(eq=False, init=False) +class Fp4MlaState(MetadataExtension): """State shared by RoPE/cache append and all FP4 FMHAs in one batch. Allocate through the metadata buffer allocator so eager forwards and each @@ -27,6 +36,55 @@ class Fp4MlaState: layer-local: MTP and metadata preparation update it before any FMHA runs. """ + fields = frozenset( + { + "batch_indices", + "cache_page_index_scale", + "cache_pool_id", + "context_repack_max_touched_pages", + "device_page_table", + "device_page_table_valid", + "full_pages_cache", + "generation_append_lens", + "generation_cache_scattered", + "generation_kv_lens", + "generation_lengths_capture_recorded", + "generation_lengths_num_contexts", + "generation_lengths_num_seqs", + "generation_lengths_num_tokens", + "hp_page_index_scale", + "hp_page_indices", + "hp_pool", + "hp_pool_id", + "kv_global_scale", + "num_blocks", + "num_context_blocks", + "num_generation_blocks", + "num_sequences", + "packed_plan_key", + "page_table_stride", + "paged_kv_indices", + "paged_kv_indptr", + "paged_kv_indptr_decode", + "positions", + "prequantized_q", + "prequantized_q_rows_per_seq", + "prequantized_q_sf", + "q_batch_capacity", + "q_global_scale", + "v_packed_cache_tags", + "v_scale_pool", + "workspaces", + } + ) + methods = frozenset( + { + "invalidate_generation_lengths", + "populate_append_metadata", + } + ) + + metadata: TrtllmAttentionMetadata | None = None hp_pool: torch.Tensor | None = None hp_page_indices: torch.Tensor | None = None v_scale_pool: torch.Tensor | None = None @@ -56,7 +114,8 @@ class Fp4MlaState: prequantized_q_sf: torch.Tensor | None = None q_batch_capacity: int | None = None prequantized_q_rows_per_seq: int | None = None - fp8_context_state: tuple[_Fp8MlaContextScratch, TrtllmAttentionMetadata] | None = None + # The FP8 context view of this step, reused by every local layer. + _fp8_context_view: tuple[_Fp8MlaContextScratch, TrtllmAttentionMetadata] | None = None _paged_kv_indptr: torch.Tensor | None = None paged_kv_indptr_decode: torch.Tensor | None = None _paged_kv_indices: torch.Tensor | None = None @@ -80,8 +139,15 @@ def paged_kv_indptr(self) -> torch.Tensor: raise RuntimeError("FP4 MLA paged_kv_indptr is not allocated.") return self._paged_kv_indptr[: self.num_sequences + 1] - @classmethod - def create(cls, metadata: TrtllmAttentionMetadata, buffers: Buffers | None) -> Fp4MlaState: + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + for f in fields(self): + if f.name == "metadata": + continue + default = f.default_factory() if f.default_factory is not MISSING else f.default + setattr(self, f.name, default) + buffers = metadata.cuda_graph_buffers + metadata = self.metadata manager = metadata.kv_cache_manager hp_ring_size = manager.fp4_mla_hp_pool_size if hp_ring_size < HP_BLOCK_SIZE: @@ -117,7 +183,9 @@ def allocate( capture_graph=metadata.is_cuda_graph, ) - state = cls(hp_pool=hp_pool, v_scale_pool=manager.get_mla_v_scale_pool()) + state = self + state.hp_pool = hp_pool + state.v_scale_pool = manager.get_mla_v_scale_pool() tokens = (metadata.max_num_tokens,) sequences = (metadata.max_num_sequences,) indptr = (metadata.max_num_sequences + 1,) @@ -134,7 +202,6 @@ def allocate( state.q_global_scale.fill_(FP4_MLA_Q_GLOBAL_SCALE) state.kv_global_scale = allocate("kv_global_scale", (1,), torch.float32) state.kv_global_scale.fill_(FP4_MLA_KV_GLOBAL_SCALE) - return state def invalidate_generation_lengths(self) -> None: self.generation_lengths_num_tokens = -1 @@ -143,9 +210,21 @@ def invalidate_generation_lengths(self) -> None: self.generation_lengths_capture_recorded = False self.packed_plan_key = None - def prepare(self, metadata: TrtllmAttentionMetadata, kv_lens: torch.Tensor) -> None: + def prepare(self) -> None: + from .metadata import configure_fp4_mla_device_page_table + + metadata = self.metadata + num_seqs = metadata.num_seqs + # FP4 MLA needs the per-forward append lengths rather than the original + # prompt lengths. Context scratch-cache metadata and MTP generation + # both consume these runtime views. + metadata.substitute_runtime_views( + prompt_lens_cuda=metadata.seq_lens_kv_cuda[:num_seqs], + prompt_lens_cpu=metadata.seq_lens_kv[:num_seqs], + ) + kv_lens = metadata.kv_lens_runtime # All local layers reuse this FP8 view only within the current batch. - self.fp8_context_state = None + self._fp8_context_view = None self.invalidate_generation_lengths() if metadata.kv_cache_manager is None or metadata.request_ids is None: raise RuntimeError( @@ -156,35 +235,38 @@ def prepare(self, metadata: TrtllmAttentionMetadata, kv_lens: torch.Tensor) -> N "FP4 MLA requires fixed-stride device page metadata; the " "current KV-cache manager or batch layout is unsupported." ) - # Graph capture's on_update_kv_lens owns the append kernel. Uniform - # generation derives token positions in the fused update instead. - if not metadata.is_cuda_graph and metadata.num_tokens > 0 and metadata.num_contexts > 0: + # Uniform generation derives token positions in the fused update + # instead. + if metadata.num_tokens > 0 and metadata.num_contexts > 0: self.populate_append_metadata(metadata) - def on_update_kv_lens(self, metadata: TrtllmAttentionMetadata) -> None: + def _refresh_length_state(self) -> None: + metadata = self.metadata self.device_page_table_valid = False if metadata.num_tokens > 0: self.invalidate_generation_lengths() if metadata.num_contexts > 0: self.populate_append_metadata(metadata) - def update_for_spec_dec(self, metadata: TrtllmAttentionMetadata) -> None: - num_seqs = metadata.num_seqs - metadata.prompt_lens_cuda_runtime = metadata.seq_lens_kv_cuda[:num_seqs] - if not torch.cuda.is_current_stream_capturing(): - metadata.prompt_lens_cpu_runtime = metadata.seq_lens_kv[:num_seqs] - self.on_update_kv_lens(metadata) - - def restore_from_spec_dec(self, metadata: TrtllmAttentionMetadata) -> None: - # Rebind aliases to the restored stable tensors, never to the temporary - # spec-dec clone whose storage may be released after graph capture. + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + # A draft step's query lengths are this draft's own: the kernels read + # them as the prompt lengths. + metadata = self.metadata num_seqs = metadata.num_seqs - metadata.kv_lens_cuda_runtime = metadata.kv_lens_cuda[:num_seqs] - metadata.prompt_lens_cuda_runtime = metadata.seq_lens_kv_cuda[:num_seqs] - if not torch.cuda.is_current_stream_capturing(): - metadata.prompt_lens_cpu_runtime = metadata.seq_lens_kv[:num_seqs] + metadata.substitute_runtime_views( + prompt_lens_cuda=metadata.seq_lens_kv_cuda[:num_seqs], + prompt_lens_cpu=( + None + if torch.cuda.is_current_stream_capturing() + else metadata.seq_lens_kv[:num_seqs] + ), + ) + self._refresh_length_state() def populate_append_metadata(self, metadata: TrtllmAttentionMetadata) -> None: + from .metadata import populate_fp4_mla_append_metadata + num_seqs = metadata.num_contexts + metadata.num_generations if num_seqs == 0 or metadata.num_tokens == 0: return diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py index 5c92b130ea0b..32035bfd5230 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py @@ -26,6 +26,7 @@ ) from .fp4_mla_kernels import _fp4_mla_rebuild_v_scale_from_k_scale_kernel from .layout import _ensure_workspace_tensor, get_fp4_mla_v_scale_pool_size +from .state import find_fp4_mla def _shared_v_pack_storage_enabled() -> bool: @@ -255,7 +256,7 @@ def _set_triton_v_packed_cache_valid( if _shared_v_pack_storage_enabled() else _triton_v_packed_valid_attr(layer_idx) ) - metadata.fp4_mla_state.v_packed_cache_tags[valid_attr] = _triton_v_packed_cache_tag( + find_fp4_mla(metadata).v_packed_cache_tags[valid_attr] = _triton_v_packed_cache_tag( layer_idx, kv_cache, v_head_dim=v_head_dim, @@ -284,7 +285,7 @@ def _is_triton_v_packed_cache_valid( if _shared_v_pack_storage_enabled() else _triton_v_packed_valid_attr(layer_idx) ) - return metadata.fp4_mla_state.v_packed_cache_tags.get(valid_attr) == _triton_v_packed_cache_tag( + return find_fp4_mla(metadata).v_packed_cache_tags.get(valid_attr) == _triton_v_packed_cache_tag( layer_idx, kv_cache, v_head_dim=v_head_dim, @@ -322,7 +323,7 @@ def _get_triton_v_packed_cache( page_ids=page_ids, ): return None - v_packed = metadata.fp4_mla_state.workspaces.get(_triton_v_packed_attr(layer_idx)) + v_packed = find_fp4_mla(metadata).workspaces.get(_triton_v_packed_attr(layer_idx)) expected_shape = _v_packed_shape(kv_cache, v_head_dim, page_size, block_v) if ( v_packed is None diff --git a/tensorrt_llm/_torch/attention/backends/interface.py b/tensorrt_llm/_torch/attention/backends/interface.py index 2e609acd69e7..38c08f1dfc72 100644 --- a/tensorrt_llm/_torch/attention/backends/interface.py +++ b/tensorrt_llm/_torch/attention/backends/interface.py @@ -1,47 +1,133 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import contextlib -import copy import weakref from collections import namedtuple from dataclasses import dataclass, field from enum import Enum, IntEnum -from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Literal, Optional, - Protocol, Tuple, Type, TypeVar, Union) +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, FrozenSet, Generic, + List, Optional, Protocol, Sequence, Tuple, Type, TypeVar, + Union) import torch from typing_extensions import Self if TYPE_CHECKING: - from ...attention.backends.prims_ts.vc_attention.quantization import \ - VCAttentionOperands + from ...attention.backends.prims_ts.vc_attention.quantization import VCAttentionOperands from ...model_config import ModelConfig - from ...modules.mamba.mamba2_metadata import Mamba2Metadata - from ...pyexecutor.kv_cache.mixed_precision_staging import \ - MixedPrecisionKVStaging + from ...pyexecutor.llm_request import LlmRequest from ...speculative.interface import SpecMetadata from ...speculative.spec_tree_manager import SpecTreeManager from ..kernels.sage_quant import SageQuantized from .fmha.gqa_decode import KvAppendPlan -from tensorrt_llm._utils import get_hf_rope_theta, maybe_pin_memory +from tensorrt_llm._utils import get_hf_rope_theta from tensorrt_llm.functional import (AttentionMaskType, PositionEmbeddingType, RopeEmbeddingUtils, RotaryScalingType) from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig -from ...memory_buffer_utils import Buffers, get_memory_buffers +from ...host_staging import copy_host_to_device +from ...memory_buffer_utils import Buffers from ...metadata import KVCacheParams from ...pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from ...pyexecutor.kv_cache.mamba_cache_manager import BaseMambaCacheManager from ...pyexecutor.resource_manager import KVCacheManager from ...pyexecutor.trace_log_utils import log_tensor_size from ...utils import get_model_extra_attrs +from .core import AttentionRuntimeFeatures # noqa: F401 (public import path) +from .core import AttentionCoreMetadata from .cpp_schema import cpp_field -from .sparse.params import (SparseBackendForwardArgs, SparseMetadataParams, - SparseRuntimeParams) +from .extensions import (ExtensionRegistry, MetadataExtension, init_field_owner, + resolve_extension_order) +from .sparse.params import SparseBackendForwardArgs, SparseRuntimeParams + +ExtensionT = TypeVar("ExtensionT", bound=MetadataExtension) +_NO_EXTENSIONS = ExtensionRegistry() +_ASSIGNABLE_NAMES: Dict[type, FrozenSet[str]] = {} +# Composed metadata classes keyed by (core class, attached extension types). +_COMPOSED_CLASSES: Dict[Tuple[type, Tuple[type, ...]], type] = {} + + +def _forwarded_field(extension_type: type, name: str) -> property: + """A property reading and writing ``name`` on the attached ``extension_type``.""" + + def get(self): + return getattr(self.__dict__["_extensions"]._by_type[extension_type], + name) + + def set(self, value): + setattr(self.__dict__["_extensions"]._by_type[extension_type], name, + value) + + return property(get, set, doc=f"{extension_type.__name__}.{name}") + + +def _forwarded_method(extension_type: type, name: str) -> property: + """A property returning the attached ``extension_type``'s bound ``name``.""" + + def get(self): + return getattr(self.__dict__["_extensions"]._by_type[extension_type], + name) + + return property(get, doc=f"{extension_type.__name__}.{name}()") + + +def _composed_class(core: type, extension_types: Tuple[type, ...]) -> type: + """``core`` with each extension's declared fields and methods forwarded. + + One class per (core, extension types) combination, so ``isinstance`` + against the core family holds and ``copy.copy`` keeps the forwarding. + """ + key = (core, extension_types) + composed = _COMPOSED_CLASSES.get(key) + if composed is None: + namespace = {"_core_class": core, "__module__": core.__module__} + core_names = _assignable_names(core) + for extension_type in extension_types: + fields = extension_type.declared_fields() + methods = extension_type.declared_methods() + if extension_type.is_core: + # A metadata class that defines one of its core's names itself + # (a test double) keeps its own definition. + fields = {name for name in fields if not hasattr(core, name)} + methods = {name for name in methods if not hasattr(core, name)} + else: + shadowed = (fields | methods) & core_names + if shadowed: + raise ValueError( + f"{extension_type.__name__} declares {sorted(shadowed)}, " + f"which {core.__name__} already defines") + for name in fields: + namespace[name] = _forwarded_field(extension_type, name) + for name in methods: + namespace[name] = _forwarded_method(extension_type, name) + names = ",".join(t.__name__ for t in extension_types) + composed = type(f"{core.__name__}[{names}]", (core, ), namespace) + _COMPOSED_CLASSES[key] = composed + return composed + + +def _assignable_names(cls: type) -> FrozenSet[str]: + """Attribute names an instance of ``cls`` may assign. + + Every name a class in the MRO declares (dataclass fields, annotations, + class attributes, properties, methods) plus each class's + ``_instance_attrs``. + """ + names = _ASSIGNABLE_NAMES.get(cls) + if names is None: + collected = set() + for klass in cls.__mro__: + if klass is object: + continue + collected.update(vars(klass)) + collected.update(vars(klass).get("__annotations__", ())) + collected.update(vars(klass).get("_instance_attrs", ())) + names = _ASSIGNABLE_NAMES[cls] = frozenset(collected) + return names + try: # Transformers v5 @@ -52,17 +138,6 @@ ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES -@dataclass -class AttentionRuntimeFeatures: - chunked_prefill: bool = False - cache_reuse: bool = False - has_speculative_draft_tokens: bool = False - # This is the chunk size for MLA chunked prefill, which splits KV cache into chunks. - chunk_size: int = 0 - # The real chunk size for MLA chunked prefill is this value * chunk_size. - chunked_prefill_buffer_batch_size: int = 4 - - # The type of requests in qkv passed to attention # Please keep sync with AttentionInputType in cpp/tensorrt_llm/thop/attentionOp.cpp class AttentionInputType(IntEnum): @@ -72,437 +147,478 @@ class AttentionInputType(IntEnum): @dataclass(kw_only=True) -class AttentionMetadata: - """ - Metadata for the attention module. +class BatchInputs: + """One step's batch description, applied by ``AttentionMetadata.prepare``. + + ``q_offsets`` / ``kv_offsets`` are optional packed-token boundaries + ``[num_seqs + 1]`` (int32); None means ``cumsum(seq_lens)``. Explicit + offsets may leave gaps between sequences (``offsets[i + 1] - offsets[i] >= + seq_lens[i]``); only metadata whose ``supports_explicit_offsets`` is True + honors offsets that differ from ``cumsum(seq_lens)``. """ - # The max number of requests in a single batch. - max_num_requests: int - # The max number of tokens in all requests in a single batch. - max_num_tokens: int - # The max number of sequences in a single batch. - max_num_sequences: Optional[int] = None - # The KV cache manager. - kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2, None] = None - # Draft KV cache manager for one-model speculative decoding with separate KV cache layouts - draft_kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2, None] = None - mapping: Optional[Mapping] = None - # Sparse settings for metadata allocation/update; dense metadata leaves it None. - sparse_metadata_params: Optional[SparseMetadataParams] = None - # Paged KV-cache block layout: - # NHD: [max_num_pages, 2, page_size, num_kv_heads, head_dim] - # HND: [max_num_pages, 2, num_kv_heads, page_size, head_dim] - kv_layout: Literal["NHD", "HND"] = "HND" - - enable_flash_mla: bool = False - enable_context_mla_with_cached_kv: bool = False - # Whether CUDA graph is enabled. - is_cuda_graph: bool = field(default=False, repr=False) - - # The length of each sequence in the batch for query. - # The shape is (batch_size), and located on CPU memory. - # For sub metadata of cross attention, it's automatically - # initialized to seq_lens of parent metadata. - seq_lens: Optional[torch.Tensor] # Implemented using property - - # The number of context-phase sequences in the batch. - num_contexts: int # Implemented using property - - # The position of each token in each sequence. - # May be None if positional embedding is applied outside the backend. - position_ids: Optional[torch.Tensor] = None - - # The number of context-phase sequences in the batch. - _num_contexts: int = field(init=False, default=0, repr=False) - # The parameters for the KV cache. + # Per-sequence query lengths [num_seqs] on the host; None keeps the bound + # lengths (CUDA graph metadata, whose lengths the graph runner owns). + seq_lens: Optional[torch.Tensor] + num_contexts: int + request_ids: List[int] + prompt_lens: List[int] kv_cache_params: Optional[KVCacheParams] = None + beam_width: int = 1 + num_chunked_ctx_requests: int = 0 + q_offsets: Optional[torch.Tensor] = None + kv_offsets: Optional[torch.Tensor] = None + # A device copy of ``seq_lens`` to bind as it is instead of staging one, + # shared with the metadata that owns it (a cross-attention sub-metadata + # shares its decoder's query lengths). + seq_lens_cuda: Optional[torch.Tensor] = None + # Per-sequence KV lengths on the host when they differ from the query + # lengths (cross attention); None keeps the bound ones. + seq_lens_kv: Optional[torch.Tensor] = None + # The step's inputs of the cross-attention sub-metadata, which prepares + # its own state from them. + cross: Optional["BatchInputs"] = None + # The step's scheduled context requests, which extensions read in their + # prepare, and whether context forwards may run in CUDA graphs. + context_requests: Optional[List["LlmRequest"]] = None + prefill_cuda_graphs: bool = False + # Per-sequence corrections [num_seqs] (int32, device) to the KV lengths the + # host-side ``kv_cache_params`` imply, known only on device when the step + # is prepared: lengths a previous step's forward changed after it was + # scheduled (accepted draft tokens, a draft rewind). Only metadata whose + # ``supports_kv_lens_offsets`` is True applies them. + kv_lens_offsets: Optional[torch.Tensor] = None + # Per-token corrections [num_tokens - num_ctx_tokens] (int32, device) to + # the positions of the generation tokens the host packed, from the same + # source as ``kv_lens_offsets``; Helix derives its rank-local lengths from + # the corrected positions. + position_id_offsets: Optional[torch.Tensor] = None + # Per-request part lengths of multi-item scoring; None when the batch + # does not score multiple items per request. + multi_item_part_lens: Optional[List[List[int]]] = None - # The length of each sequence in the batch for key and value. - # The shape is (batch_size), and located on CPU memory. - # It defaults to seq_lens if not set. - # Should only set explicitly for cross attention. - seq_lens_kv: Optional[torch.Tensor] # Implemented using property - - # Actual storage for seq_lens and seq_lens_kv - _seq_lens: Optional[torch.Tensor] = field(init=False, - repr=False, - default=None) - _seq_lens_kv: Optional[torch.Tensor] = field(init=False, - repr=False, - default=None) - - # A copy of seq_lens store on the GPU. Used in the logits - # processor. Using 2 copies avoids a lot of extraneous - # copies in flashinfer's prepare() implementation. - _seq_lens_cuda: Optional[torch.Tensor] = field(init=False, - repr=False, - default=None) - _seq_lens_kv_cuda: Optional[torch.Tensor] = field(init=False, - repr=False, - default=None) - - # For self attention, this is the sub metadata for cross attention - # that works together in one model. - # For cross attention, this is automatically inited to self, - # and must not be set explicitly. - cross: Optional["AttentionMetadata"] = None - - # The request ID of each sequence in the batch. - # The shape is (batch_size). - request_ids: Optional[List[int]] = None - - # The prompt length of each sequence in the batch. - # For context-phase sequence, the value is its token number, which is same with `context_lens`. - # For generation-phase sequence, the value is the token number of its context phase. - # The shape is (batch_size) if provided. - prompt_lens: Optional[List[int]] = None - - # Explicit query/KV sequence boundaries for attention kernels that operate - # on packed varlen context inputs. These are sequence/segment boundaries, - # not necessarily request boundaries. - cu_q_seqlens: Optional[torch.Tensor] = None - cu_kv_seqlens: Optional[torch.Tensor] = None - - # These fields indicate whether the runtime can use various features. - # The kernels may or may not have different behaviors when these - # are enabled. - runtime_features: AttentionRuntimeFeatures = field( - default_factory=AttentionRuntimeFeatures) - - # The number of tokens in each rank. - all_rank_num_tokens: Optional[List[int]] = None - - def get_adp_token_counts(self) -> List[int]: - """Local execution-stage counts carried by the input count exchange.""" - return [self.num_tokens] - - def set_adp_token_counts(self, counts: List[List[int]]) -> List[int]: - """Save stage counts and return the input-stage counts for padding.""" - return counts[0] - - # These fields are set when changing seq_lens and _num_contexts to avoid computation - # during execution. If the calculation happens during execution, torch compile treats it - # as DDS and fails to compile. - _num_generations: int = field(init=False, default=0, repr=False) - _num_ctx_tokens: int = field(init=False, default=0, repr=False) - _num_tokens: int = field(init=False, default=0, repr=False) - mamba_metadata: Optional[Any] = None - mamba_chunk_size: int = 128 - mamba_metadata_cls: Optional[Type["Mamba2Metadata"]] = None - - # The number of tokens in the padded sequence. - padded_num_tokens: Optional[int] = None - - # This buffer is currently only used for TrtllmAttentionMetadata. - cache_indirection: Optional[torch.Tensor] = None - cuda_graph_buffers: dict[str, list[torch.Tensor]] = None - - _saved_tensors: Dict[str, torch.Tensor] = field(init=False, - default_factory=dict) - # Upper bound on query heads per KV head over the target model's layers - # (unsharded); sizes the spec-dec tree mask. - max_num_heads_per_kv: Optional[int] = 1 +class AttentionMetadata: + """A container of the extensions that hold an attention metadata's state. - multi_item_part_lens: Optional[list[list[int]]] = None - """Additional token layout information for multi-item scoring. + The container holds no state of its own: its family's core + (``core_metadata_type``, always attached first) holds the batch layout and + the family's buffers, and every feature, sparse algorithm or model adds an + extension. The container drives their lifecycle and forwards their + declared fields and methods, so ``metadata.seq_lens`` reads the core's. - Aggregates `TokensPrompt.multi_item_part_lens` for all requests in the batch, - see `TokensPrompt` for details. + Assigning a name that neither the container's classes nor an attached + extension declares raises ``AttributeError``. """ - # Mixed-precision KV: the staging copy attention reads, created by the - # first prepare() on a manager with mixed-precision layers. - mixed_precision_staging: Optional["MixedPrecisionKVStaging"] = field( - init=False, default=None, repr=False, compare=False) - # The same for the MTP layers of a separate draft manager; swapped in for draft forwards. - draft_mixed_precision_staging: "MixedPrecisionKVStaging | None" = field( - init=False, default=None, repr=False, compare=False) + # Names a test double defined on a subclass may assign; the families keep + # their state in their core. + _instance_attrs: ClassVar[FrozenSet[str]] = frozenset() + # The core extension holding this family's batch layout and state. + core_metadata_type: ClassVar[ + Type[AttentionCoreMetadata]] = AttentionCoreMetadata + # Whether ``prepare`` applies ``BatchInputs.kv_lens_offsets`` to the + # device KV lengths before the extensions prepare. + supports_kv_lens_offsets: ClassVar[bool] = False + # Whether the kernels read ``cu_q_seqlens`` / ``cu_kv_seqlens`` as given, + # so they may differ from cumsum(seq_lens). Metadata that derives the + # boundaries itself rejects offsets that differ. + supports_explicit_offsets: ClassVar[bool] = False + + # Extensions a backend's metadata class attaches to every instance; set on + # the class AttentionBackend derives from its ``metadata_extensions``. + backend_extension_types: ClassVar[Tuple[Type[MetadataExtension], ...]] = () + + # The attached extensions, the core first; see extensions.py. + _extensions: ExtensionRegistry + + def __init__(self, + *, + extension_types: Sequence[Type[MetadataExtension]] = (), + copy_of: Optional["AttentionMetadata"] = None, + **kwargs: Any) -> None: + """Construct the core and the extensions, routing each argument to its owner. + + Every keyword argument is a field some extension takes (its + ``init_fields``). The extensions attached are the family's defaults, + chosen from the arguments, plus ``extension_types``. An argument whose + owner is not attached is an error unless it is None, which means not + given; an argument no extension takes is an error. + + Each extension's constructor allocates its buffers, in dependency + order and the core first, so it can read the extensions constructed + before it. ``copy_of`` makes this metadata a copy of another (see + ``constructed_copy``): every extension takes its declared ``shared`` + attributes from that metadata's. + """ + cls = type(self) + core_type = cls.core_metadata_type + # The backend's extensions follow the family's, whose state (such as + # the speculative-decoding settings) they read when constructed. + types = resolve_extension_order([ + *cls.default_extension_types(kwargs), *cls.backend_extension_types, + *extension_types + ]) + # Extensions that need the complete metadata come last. + types = ([t for t in types if not t.constructed_last] + + [t for t in types if t.constructed_last]) + owners: Dict[str, type] = {} + for owner in (core_type, *types): + for name in owner.declared_init_fields(): + owners.setdefault(name, owner) + routed: Dict[type, Dict[str, Any]] = {} + for name, value in kwargs.items(): + owner = owners.get(name) + if owner is not None: + routed.setdefault(owner, {})[name] = value + continue + elsewhere = init_field_owner(name) + if elsewhere is None: + raise TypeError(f"{cls.__name__}() got an unexpected keyword " + f"argument {name!r}") + if value is not None: + raise TypeError( + f"{cls.__name__}() got {name!r}, which " + f"{elsewhere.__name__} takes, but does not attach it") + missing = [ + name for name in core_type.required_init_fields + if name not in routed.get(core_type, {}) + ] + if missing: + raise TypeError(f"{cls.__name__}() missing required keyword " + f"arguments {missing}") + sources = ({} if copy_of is None else { + type(e): e + for e in copy_of._attached_extensions() + }) + registry = ExtensionRegistry(self) + self.__dict__["_extensions"] = registry + registry.attach_new_core(self, core_type, routed.get(core_type, {}), + sources.get(core_type)) + registry.attach(self, types, routed, sources) + + def constructed_copy(self, + *, + share: bool = True, + without: Sequence[Type[MetadataExtension]] = (), + **changes: Any) -> Self: + """A new metadata built from this one's configuration, with ``changes``. + + Every extension attached now is attached to the copy, constructed with + this metadata's current creation arguments (each extension's + ``creation_config``) updated by ``changes``; the copy allocates its own + buffers from its pool and, with ``share``, takes the ``shared`` + attributes from this metadata's extensions. Extensions of the types + in ``without`` are left out. For CUDA graph, cross-attention and draft + metadata. + """ + config: Dict[str, Any] = {} + for extension in self._attached_extensions(): + if not isinstance(extension, tuple(without)): + config.update(extension.creation_config()) + config.update(changes) + types = [ + extension_type + for extension_type in self._attached_extensions().types() + if not extension_type.is_core + and not issubclass(extension_type, tuple(without)) + ] + return self.core_class()(extension_types=types, + copy_of=self if share else None, + **config) - def __post_init__(self) -> None: - if self.is_cross: - assert self.cross is None or self.cross is self, "Cross attention metadata should not have sub metadata" - self.cross = self - return - - assert self.cross is None or type(self) is type( - self.cross - ), "Top level and cross attention sub metadata type mismatched" - - def on_update_kv_lens(self): - pass - - def on_update(self): - if (self._seq_lens is not None - and self._seq_lens.shape[0] >= self.num_contexts - and self.num_contexts >= 0): - self._num_ctx_tokens = self._seq_lens[:self.num_contexts].sum( - ).item() - self._num_generations = self._seq_lens.shape[0] - self.num_contexts - if self._seq_lens_kv is not None: - self._num_tokens = self._seq_lens_kv.sum().item() - elif self._seq_lens is not None: - self._num_tokens = self._seq_lens.sum().item() + @classmethod + def default_extension_types( + cls, settings: Dict[str, Any]) -> List[Type[MetadataExtension]]: + """Extensions this family attaches, chosen from the constructor arguments.""" + from .features import CrossAttentionMetadata, DraftMetadata + types: List[Type[MetadataExtension]] = [DraftMetadata] + if isinstance(settings.get("kv_cache_manager"), BaseMambaCacheManager): + from .features.recurrent import RecurrentMetadata + types.append(RecurrentMetadata) + if (settings.get("cross") is not None + or settings.get("seq_lens_kv") is not None): + types.append(CrossAttentionMetadata) + return types + + def __setattr__(self, name: str, value: Any) -> None: + if name not in _assignable_names(type(self)): + if self._routes_to_core(name): + # A metadata built without __init__ (as some tests do) gets + # its core first; the composed class then forwards the write. + self._ensure_core() + setattr(self, name, value) + return + raise AttributeError( + f"{type(self).__name__} declares no attribute {name!r}; keep " + "feature or model state in a MetadataExtension") + object.__setattr__(self, name, value) + + def __getattr__(self, name: str) -> Any: + # Reached only when normal lookup fails: route a core field of a + # metadata built without __init__ (as some tests do) to its core. + if self._routes_to_core(name): + self._ensure_core() + return getattr(self, name) + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}") + + def _routes_to_core(self, name: str) -> bool: + if name.startswith("__"): + return False + registry = self.__dict__.get("_extensions") + if registry is not None and registry.core() is not None: + return False + return name in type(self).core_metadata_type.declared_names() + + def __copy__(self) -> Self: + """A shallow copy that owns a copy of the core state. + + The other extensions stay shared until the copy binds its own registry + (``bound_to``), as with any ``copy.copy`` view of the metadata. + """ + clone = object.__new__(type(self)) + clone.__dict__.update(self.__dict__) + registry = self.__dict__.get("_extensions") + if registry is not None: + clone.__dict__["_extensions"] = registry.with_core_copied(clone) + return clone + + def core(self) -> AttentionCoreMetadata: + """This metadata's core: its batch layout and family state.""" + return self._ensure_core() + + def _ensure_core(self) -> AttentionCoreMetadata: + """Bind this metadata's registry and attach its core first, if missing.""" + registry = self.__dict__.get("_extensions") + if registry is None or registry.owner is not self: + registry = (registry or _NO_EXTENSIONS).bound_to(self) + self.__dict__["_extensions"] = registry + core = registry.core() + if core is None: + # A metadata made without __init__ (as some tests do) gets its + # family's core with the default configuration: no capacities. + core = registry.attach_new_core(self, + type(self).core_metadata_type, {}, + None) + return core - @property - def seq_lens(self) -> Optional[torch.Tensor]: - return self._seq_lens - - @seq_lens.setter - def seq_lens(self, value: Optional[torch.Tensor]): - # If value not explicitly given, dataclass tries to initialize using class attribute - value = value if value is not AttentionMetadata.seq_lens else None - self._seq_lens = value - self.on_update() - - # The model executor sets seq_lens to None initially. - if self._seq_lens is not None: - self._seq_lens = maybe_pin_memory(self._seq_lens) - - if self.is_cuda_graph and self._seq_lens_cuda is not None: - # Very important: do not reallocate if we are using CUDA graphs. - # This copy is safe because the batch size is guaranteed to not - # change in the CUDA graph case. The seqlens can change if we - # are doing spec decode. - self._seq_lens_cuda.copy_(self._seq_lens, non_blocking=True) - else: - self._seq_lens_cuda = self._seq_lens.cuda(non_blocking=True) + def get_adp_token_counts(self) -> List[int]: + """Local execution-stage counts carried by the input count exchange. - if self.has_cross_sub_metadata: - self.cross._seq_lens = self._seq_lens - self.cross._seq_lens_cuda = self._seq_lens_cuda + The first stage is the input (``num_tokens``, unless an extension + declares ``adp_input_token_count``); each extension may append its own. + """ + input_tokens = self.find("adp_input_token_count", None) + if input_tokens is None: + input_tokens = self.num_tokens + return [input_tokens, *(self.find("adp_token_counts", None) or ())] - @property - def num_contexts(self) -> int: - return self._num_contexts + def set_adp_token_counts(self, counts: List[List[int]]) -> List[int]: + """Save stage counts and return the input-stage counts for padding.""" + if len(counts) > 1: + self.find("receive_adp_token_counts")(counts[1:], counts[0]) + return counts[0] - @num_contexts.setter - def num_contexts(self, value: int): - value = value if value is not AttentionMetadata.num_contexts else 0 - self._num_contexts = value - self.on_update() + def find(self, name: str, default: Any = None) -> Any: + """An attached extension's declared field or bound method, or ``default``. - @property - def num_generations(self) -> int: - return self._num_generations + For values the core or the engine reads from whichever extension + provides them, such as ``kv_cache_pools`` or ``full_block_table_width``. + """ + for extension in self._attached_extensions(): + declared = type(extension) + if name in declared.declared_fields( + ) or name in declared.declared_methods(): + return getattr(extension, name) + return default - @num_generations.setter - def num_generations(self, value: int): - value = value if value is not AttentionMetadata.num_generations else 0 - self._num_generations = value - self.on_update() + @classmethod + def core_class(cls) -> type: + """The metadata family class, without the forwarding of a composed class.""" + return cls.__dict__.get("_core_class", cls) - @property - def seq_lens_cuda(self): - return self._seq_lens_cuda + def _compose(self) -> None: + """Switch to the composed class of the attached extensions. - @property - def seq_lens_kv(self) -> Optional[torch.Tensor]: - return self._seq_lens_kv if self._seq_lens_kv is not None else self._seq_lens - - @seq_lens_kv.setter - def seq_lens_kv(self, value: Optional[torch.Tensor]): - value = value if value is not AttentionMetadata.seq_lens_kv else None - self._seq_lens_kv = value - self.on_update() - # The model executor sets seqlens to None initially. - if self._seq_lens_kv is not None: - self._seq_lens_kv = maybe_pin_memory(self._seq_lens_kv) - if self.is_cuda_graph and self._seq_lens_kv_cuda is not None: - self._seq_lens_kv_cuda.copy_(self._seq_lens_kv, - non_blocking=True) - else: - self._seq_lens_kv_cuda = self._seq_lens_kv.cuda( - non_blocking=True) + The composed class forwards each extension's declared fields and + methods, so callers read ``metadata.field`` instead of + ``metadata.ext(T).field``. + """ + extension_types = tuple(self._attached_extensions().types()) + object.__setattr__(self, "__class__", + _composed_class(self.core_class(), extension_types)) + + def own_attribute_names(self) -> FrozenSet[str]: + """Names this metadata's family defines; extensions may not shadow them.""" + return _assignable_names(self.core_class()) + + def _attached_extensions(self) -> ExtensionRegistry: + # Metadata built without __init__ (as some tests do) has no registry. + return self.__dict__.get("_extensions", _NO_EXTENSIONS) + + def ext(self, cls: Type[ExtensionT]) -> ExtensionT: + """The attached extension of type ``cls``; raises when none is.""" + found = self._attached_extensions().find(cls) + if found is None: + raise LookupError( + f"{type(self).__name__} has no {cls.__name__} extension attached." + ) + return found - @property - def seq_lens_kv_cuda(self): - return self._seq_lens_kv_cuda if self._seq_lens_kv_cuda is not None else self._seq_lens_cuda + def find_ext(self, cls: Type[ExtensionT]) -> Optional[ExtensionT]: + """The attached extension of type ``cls``, or None.""" + return self._attached_extensions().find(cls) - @property - def context_lens(self) -> torch.Tensor: - """ - The length of each context-phase query sequence in the batch. - The shape is (num_contexts), where num_contexts is the number of context-phase sequences in the batch. - """ - return self.seq_lens[:self.num_contexts] + def add_ext(self, cls: Type[ExtensionT]) -> ExtensionT: + """Construct and attach ``cls`` and its requirements if absent; return the extension.""" + self._ensure_core() + self._extensions.attach(self, (cls, )) + return self.ext(cls) - @property - def num_seqs(self) -> int: + def replace_ext(self, extension: MetadataExtension) -> None: + """Attach a built extension in place of any of its type. + + For derived views made by ``copy.copy``: binds this metadata to its own + registry first, so the source keeps its extensions. """ - The number of sequences in the batch. + assert extension.metadata is self, "the extension must be built for this metadata" + if not extension.is_core: + self._ensure_core() + elif self.__dict__.get( + "_extensions") is None or self._extensions.owner is not self: + self._extensions = self._attached_extensions().bound_to(self) + self._extensions.replace(extension) + self._compose() + + def remove_ext(self, cls: Type[MetadataExtension]) -> None: + """Detach the extension of type ``cls`` from a derived view. + + Like ``replace_ext``, binds this metadata to its own registry first. """ - return self.seq_lens.shape[0] + self._ensure_core() + self._extensions.remove(cls) + self._compose() @property - def is_cross(self) -> bool: + def extensions(self) -> List[MetadataExtension]: + """The attached extensions in dependency order.""" + return list(self._attached_extensions()) + + def get_buffer(self, shape: Sequence[int], dtype: torch.dtype, + name: str) -> torch.Tensor: + """A persistent buffer from this metadata's pool, graph-stable under capture.""" + return self.get_empty(self.cuda_graph_buffers, + list(shape), + dtype=dtype, + cache_name=name, + capture_graph=self.is_cuda_graph) + + def stage_to_device(self, key: Any, destination: torch.Tensor, + value: torch.Tensor) -> None: + """Copy a host tensor into ``destination`` through pinned staging.""" + copy_host_to_device(self.core()._host_staging, key, destination, value) + + def adopts_target_state(self) -> bool: + """Whether every attached extension takes a draft's state from its target's.""" + return all( + type(extension).adopts_target_state + for extension in self._attached_extensions()) + + def check_lengths_current(self) -> None: + """Raise when the lengths changed in place without a refresh. + + The attention forward calls it: state derived from the lengths would + be stale. """ - Is this metadata for cross attention. - """ - return self.seq_lens is not self.seq_lens_kv + if self.lengths_changed: + raise RuntimeError( + f"{type(self).__name__}'s lengths changed in place since its " + "last prepare or draft refresh; refresh the draft before its " + "forward") @property def has_cross_sub_metadata(self) -> bool: - return self.cross is not None and self.cross is not self - - @property - def num_ctx_tokens(self) -> int: - return self._num_ctx_tokens - - @property - def num_tokens(self) -> int: - return self._num_tokens - - def prepare(self): - """ - Hook to be called before the forward step of the model. - """ - self._prepare_mamba_metadata() - - def _has_mixed_precision_staging(self) -> bool: - """Whether this backend runs attention on a staging copy of the mixed-precision layers.""" - # isinstance guards against mocks and against V1 managers, which have - # no mixed-precision layers. - high_layers = getattr(self.kv_cache_manager, - "mixed_precision_high_layers", None) - return isinstance(high_layers, dict) and bool(high_layers) - - def prepare_mixed_precision_kv(self, layout: str, - tokens_per_block: int) -> None: - """Create the staging copies on first use, then refresh them per batch.""" - self.mixed_precision_staging = self._refresh_mixed_precision_kv( - self.mixed_precision_staging, self.kv_cache_manager, "mixed", - layout, tokens_per_block) - draft = self.draft_kv_cache_manager - draft_high_layers = getattr(draft, "mixed_precision_high_layers", None) - # A draft view runs on the draft manager itself and has no draft of its own. - if (draft is not self.kv_cache_manager - and isinstance(draft_high_layers, dict) and draft_high_layers): - self.draft_mixed_precision_staging = self._refresh_mixed_precision_kv( - self.draft_mixed_precision_staging, draft, "draft_mixed", - layout, tokens_per_block) - - def sync_draft_mixed_precision_staging( - self) -> "MixedPrecisionKVStaging | None": - """Give the draft staging copy the target's lengths at the start of a draft forward. - - A mixed-precision MTP draft manager holds the same tokens as the target manager, - and overlap-scheduler corrections may have changed the target's lengths on the - device since prepare(). - - Returns: - The draft staging copy; None without a separate mixed-precision draft manager. + """Whether this metadata carries a cross-attention sub-metadata.""" + from .features import CrossAttentionMetadata + cross = self.find_ext(CrossAttentionMetadata) + return cross is not None and cross.has_sub_metadata + + def prepare(self, + inputs: Optional[BatchInputs] = None, + *, + context_requests: Optional[List["LlmRequest"]] = None, + prefill_cuda_graphs: bool = False) -> None: + """Prepare this step's state before the model forward. + + ``inputs`` replaces the batch description; without it the fields + already set are used. ``inputs.cross`` prepares the cross-attention + sub-metadata after this metadata. ``context_requests`` (or + ``inputs.context_requests``) are this step's scheduled context requests, + which extensions read in their own ``prepare``. + + The core derives first; every other extension then prepares in + dependency order and may correct the core's state only through core + methods. """ - draft = self.draft_mixed_precision_staging - if draft is not None: - draft.lengths.copy_(self.mixed_precision_staging.lengths) - return draft - - def _refresh_mixed_precision_kv( - self, view: "MixedPrecisionKVStaging | None", - manager: KVCacheManagerV2, name: str, layout: str, - tokens_per_block: int) -> "MixedPrecisionKVStaging": - from ...pyexecutor.kv_cache.mixed_precision_staging import \ - MixedPrecisionKVStaging - - if view is None or view.manager is not manager: - # Built once per metadata object (eager or CUDA graph copy); the - # reserved buffer-pool block is shared, so the staging copy is - # allocated and zeroed once per rank and the KV budget profiling - # run already accounts for it. - buffers = (self.cuda_graph_buffers if self.cuda_graph_buffers - is not None else get_memory_buffers()) - view = MixedPrecisionKVStaging(manager, - buffers, - rows=self.max_num_sequences - or self.max_num_requests, - layout=layout, - pool_layout=self.kv_layout, - tokens_per_block=tokens_per_block, - name=name) - logger.info_once( - f"Mixed-precision KV staging ({name}) on this rank: " - f"{view.staging_bytes / (1 << 20):.1f} MiB = {view.rows} " - f"requests x {view.max_blocks_per_seq} pages x " - f"{view.max_page_bytes} B ({layout}, {tokens_per_block} " - "tokens per page)", - key=f"{name}_precision_kv_staging") - view.update(self.request_ids, - self.kv_cache_params.num_cached_tokens_per_seq, - self.seq_lens.tolist(), - capture_graph=self.is_cuda_graph) - self._on_mixed_precision_kv_prepared(view) - return view - - def _on_mixed_precision_kv_prepared( - self, view: "MixedPrecisionKVStaging") -> None: - """Backend hook: refresh backend-specific views of the staging cache.""" - - def get_mixed_precision_kv_view( - self, layer_idx: int) -> Optional["MixedPrecisionKVStaging"]: - """Return the staging cache of a mixed-precision layer, else None.""" - staging = self.mixed_precision_staging - if (staging is None or layer_idx not in staging.pools - or staging.manager is not self.kv_cache_manager): - # No staging for a layer without a mixed-precision high pool, or of a manager not in use. - return None - return staging - - @contextlib.contextmanager - def mixed_precision_kv_staging(self, layer_idx: int): - """Run attention on the staging copy, then write the new tokens back.""" - view = self.get_mixed_precision_kv_view(layer_idx) - if view is None: - yield - return - view.convert(layer_idx) - saved = self._swap_mixed_precision_kv(view) - try: - yield - finally: - self._restore_mixed_precision_kv(saved) - view.convert(layer_idx, writeback=True) - - def _swap_mixed_precision_kv(self, - view: "MixedPrecisionKVStaging") -> tuple: - """Point this metadata at the staging cache; return what to restore.""" - saved = (self.kv_cache_manager, self.kv_layout) - self.kv_cache_manager = view - self.kv_layout = view.layout - return saved - - def _restore_mixed_precision_kv(self, saved: tuple) -> None: - """Undo _swap_mixed_precision_kv.""" - self.kv_cache_manager, self.kv_layout = saved - - def prepare_encoder_only(self) -> None: - """Hook for encoder-only (no-KV-cache) forward setup. - - Defaults to the full ``prepare()``; backends with a leaner encoder-only - path override this. + core = self.core() + if inputs is not None: + if (inputs.kv_lens_offsets is not None + and not self.supports_kv_lens_offsets): + raise ValueError( + f"{type(self).__name__} keeps no device KV lengths to " + "apply kv_lens_offsets to") + core.apply_batch_inputs(inputs) + if context_requests is None: + context_requests = inputs.context_requests + prefill_cuda_graphs = inputs.prefill_cuda_graphs + core.set_context_requests(context_requests, prefill_cuda_graphs) + if not self.supports_explicit_offsets: + self._check_offsets_match_lengths() + self._prepare_core() + for extension in self._attached_extensions(): + if extension is not core: + extension.prepare() + # Core corrections extensions made in their prepare (Rocket, Helix) + # are part of the step's derivation. + core.clear_lengths_changed() + if inputs is not None and inputs.cross is not None: + from .features import CrossAttentionMetadata + self.ext(CrossAttentionMetadata).prepare_sub_metadata(inputs.cross) + + def _check_offsets_match_lengths(self) -> None: + """Reject explicit boundaries this metadata's kernels would ignore.""" + for offsets, lengths, name in ((self.cu_q_seqlens, self.seq_lens, + "cu_q_seqlens"), + (self.cu_kv_seqlens, self.seq_lens_kv, + "cu_kv_seqlens")): + if offsets is None or lengths is None: + continue + expected = torch.zeros(lengths.numel() + 1, dtype=torch.int64) + torch.cumsum(lengths.to(torch.int64), 0, out=expected[1:]) + if not torch.equal(offsets.cpu().to(torch.int64)[:expected.numel()], + expected): + raise ValueError( + f"{type(self).__name__} derives sequence boundaries from " + f"seq_lens and cannot honor a {name} that differs from " + "their cumulative sum") + + def _prepare_core(self) -> None: + """Derive the core's per-step state; ``prepare`` runs the extensions around it.""" + self.core().derive() + + def prepare_encoder_only(self, + inputs: Optional[BatchInputs] = None) -> None: + """Encoder-only (no-KV-cache) forward setup. + + ``inputs`` replaces the batch description; without it the fields + already set are used. The core defaults to the full ``prepare()``; a + family with a leaner encoder-only path overrides it in its core. """ - self.prepare() - - def _prepare_mamba_metadata(self): - if self.mamba_metadata is False: - return - - if self.mamba_metadata is None: - if isinstance(self.kv_cache_manager, BaseMambaCacheManager): - metadata_cls = self.mamba_metadata_cls - if metadata_cls is None: - from ...modules.mamba.mamba2_metadata import Mamba2Metadata - metadata_cls = Mamba2Metadata - self.mamba_metadata = metadata_cls( - self.max_num_requests, - self.mamba_chunk_size, - max_num_tokens=self.max_num_tokens) - else: - self.mamba_metadata = False - return - - self.mamba_metadata.prepare(self) + if inputs is not None: + self.core().apply_batch_inputs(inputs) + self.core().prepare_encoder_only() + self.core().clear_lengths_changed() def create_cuda_graph_metadata(self, max_batch_size: int, @@ -522,98 +638,42 @@ def create_cuda_graph_metadata(self, if self.is_cuda_graph: return self - cuda_graph_metadata = copy.copy(self) - cuda_graph_metadata.is_cuda_graph = True - cuda_graph_metadata.cuda_graph_buffers = buffers - if self.has_cross_sub_metadata: - cuda_graph_metadata.cross = cuda_graph_metadata.cross.create_cuda_graph_metadata( - max_batch_size, True, max_draft_tokens, buffers, encode_only) - if not sub_cross_metadata: - # Set to None to force the cuda graph metadata to allocate a tensor - # with the correct batch size. See seq_lens setter for how this works. - cuda_graph_metadata._seq_lens_cuda = None - if encode_only: - # Encoder: variable seq_lens per batch; the runner in-place - # updates them via the seq_lens setter each replay. - cuda_graph_metadata.seq_lens = torch.ones((max_batch_size, ), - dtype=torch.int) - else: - cuda_graph_metadata.seq_lens = torch.ones( - (max_batch_size, ), - dtype=torch.int) * (1 + max_draft_tokens) - if self.is_cross: - cuda_graph_metadata.seq_lens_kv = torch.zeros((max_batch_size, ), - dtype=torch.int) - if self.enable_flash_mla: - if self.kv_cache_manager is not None: - cuda_graph_metadata.block_ids_per_seq = torch.zeros( - [ - self.kv_cache_manager.max_batch_size, - self.kv_cache_manager.max_blocks_per_seq - ], - dtype=torch.int32, - device='cuda', - ) - - if not encode_only: - # Decoder CUDA graphs are always generation-only (no context - # requests). Encoder CUDA graphs are the opposite (all context); - # the caller sets num_contexts = padded_batch_size. - cuda_graph_metadata.num_contexts = 0 - cuda_graph_metadata.__post_init__() + cuda_graph_metadata = self.constructed_copy( + is_cuda_graph=True, + cuda_graph_buffers=buffers, + **self.core().cuda_graph_copy_changes(max_batch_size, + max_draft_tokens)) + graph_core = cuda_graph_metadata.core() + from .features import CrossAttentionMetadata + cross = cuda_graph_metadata.find_ext(CrossAttentionMetadata) + if cross is not None: + cross.copy_sub_metadata_for_cuda_graph(max_batch_size, + max_draft_tokens, buffers, + encode_only) + graph_core.reset_cuda_graph_lengths( + max_batch_size, + max_draft_tokens=max_draft_tokens, + sub_cross_metadata=sub_cross_metadata, + encode_only=encode_only) return cuda_graph_metadata - def prepare_for_spec_dec(self, *fields) -> None: - assert len(self._saved_tensors) == 0, ( - "prepare_for_spec_dec called while fields " - f"{list(self._saved_tensors)} are still saved; a previous " - "forward likely raised between prepare_for_spec_dec and " - "restore_from_spec_dec") - for f in fields: - v = getattr(self, f) - assert isinstance(v, torch.Tensor), f"{f} is not a torch.Tensor" - self._saved_tensors[f] = v - setattr(self, f, v.clone()) - - def restore_from_spec_dec(self) -> None: - for f, v in self._saved_tensors.items(): - setattr(self, f, v) - self._saved_tensors.clear() - - @property - def has_spec_dec_saved_state(self) -> bool: - """True when prepare_for_spec_dec state has not been restored yet.""" - return bool(self._saved_tensors) - def update_spec_dec_param( self, - batch_size, - is_spec_decoding_enabled, - is_spec_dec_tree, - is_spec_dec_dynamic_tree, - max_draft_len, - max_total_draft_tokens, + batch_size: int, spec_metadata: Optional['SpecMetadata'] = None, spec_tree_manager: Optional['SpecTreeManager'] = None, - num_contexts: int = 0): - """ - Hook to be called when using TRTLLM attention backend in spec-dec mode. + num_contexts: int = 0) -> None: + """Write one step's draft-token layout; the settings are creation config. ``num_contexts`` is the number of context (prefill) requests in the mixed batch, occupying the leading rows of slot-storage buffers. Backends that consume gen-only slots (e.g. dynamic tree) must skip these rows to align with the XQA kernel's expected row layout. + Metadata without speculative-decoding state ignores it. """ - - def update_helix_param( - self, - helix_position_offsets: List[int], - helix_is_inactive_rank: List[bool], - helix_owned_new_tokens: Optional[List[int]] = None, - ) -> None: - """ - Hook to be called when using helix parallelism. - """ + update = self.find("update") + if update is not None: + update(batch_size, spec_metadata, spec_tree_manager, num_contexts) def create_cross_metadata( self, @@ -622,129 +682,15 @@ def create_cross_metadata( None] = None, encoder_num_cached_tokens_per_seq: Optional[List[int]] = None, ) -> "AttentionMetadata": - """Build a sub-metadata instance for cross-attention. - - The returned metadata shares Q-side fields (``seq_lens``, - ``request_ids``, ``num_contexts``) with ``self`` (the decoder - self-attention metadata) and overrides the K/V-side with the encoder - lengths so that ``returned.is_cross is True``. - - This is intended to be called by the runtime / unit tests once the - encoder lengths and cross-pool KV cache manager are known. The - returned object is a *new* metadata instance (not stored on - ``self.cross``); callers can attach it to ``self.cross`` if desired. - - Args: - encoder_seq_lens: Per-request encoder sequence lengths. On the - first decoder context step this is the full encoder length; - on generation steps it should be ``0`` (no new K/V tokens to - add to the cross pool — the encoder K/V are already cached). - cross_kv_cache_manager: KV cache manager for the cross pool. - When ``None``, the returned metadata uses the stateless - (no-KV-cache) path (suitable for unit tests). - encoder_num_cached_tokens_per_seq: Per-request count of encoder - K/V tokens already present in the cross pool. ``None`` - defaults to 0 (context phase, nothing cached yet). - - Returns: - A new ``AttentionMetadata`` of the same subclass as ``self``, - with ``seq_lens_kv`` set to ``encoder_seq_lens`` so that - ``is_cross`` becomes ``True``. - """ - cross_md = copy.copy(self) - cross_md._saved_tensors = {} - if self.is_cuda_graph: - # Cross-attention has K/V lengths from the encoder, while - # self-attention has K/V lengths from the decoder. Keep their - # CUDA graph metadata buffers separate so preparing cross metadata - # cannot overwrite self-attention sequence lengths. - cross_md.cuda_graph_buffers = Buffers() - cross_md._seq_lens_kv_cuda = None - cross_md.cross = None - self._update_cross_metadata( - cross_md, - encoder_seq_lens, - cross_kv_cache_manager, - encoder_num_cached_tokens_per_seq, - base_kv_cache_params=self.kv_cache_params, - block_ids_per_seq=None, - ) - cross_md.__post_init__() - return cross_md - - def _update_cross_metadata( - self, - cross_md: "AttentionMetadata", - encoder_seq_lens: List[int], - cross_kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2, None], - encoder_num_cached_tokens_per_seq: Optional[List[int]], - *, - base_kv_cache_params: Optional[KVCacheParams], - block_ids_per_seq: Optional[List[list]], - ) -> "AttentionMetadata": - encoder_seq_lens_tensor = torch.tensor(encoder_seq_lens, - dtype=torch.int) - cross_md.kv_cache_manager = cross_kv_cache_manager - cross_md._seq_lens = self.seq_lens - cross_md._seq_lens_cuda = self.seq_lens_cuda - cross_md.seq_lens_kv = encoder_seq_lens_tensor - - # Cross-attention keeps decoder-side prompt lengths for the Q-side - # context metadata. Encoder-side lengths are represented by - # seq_lens_kv and kv_cache_params.num_cached_tokens_per_seq. - cross_md.prompt_lens = self.prompt_lens - - if encoder_num_cached_tokens_per_seq is not None: - cross_md.kv_cache_params = KVCacheParams( - use_cache=(base_kv_cache_params.use_cache - if base_kv_cache_params is not None else - (cross_kv_cache_manager is not None)), - num_cached_tokens_per_seq=list( - encoder_num_cached_tokens_per_seq), - block_ids_per_seq=block_ids_per_seq, - host_max_attention_window_sizes=( - base_kv_cache_params.host_max_attention_window_sizes - if base_kv_cache_params is not None else None), - host_sink_token_length=( - base_kv_cache_params.host_sink_token_length - if base_kv_cache_params is not None else None), - num_extra_kv_tokens=(base_kv_cache_params.num_extra_kv_tokens if - base_kv_cache_params is not None else 0), - ) - - cross_md.request_ids = self.request_ids - cross_md.num_contexts = self.num_contexts - return cross_md - - def update_cross_metadata( - self, - encoder_seq_lens: List[int], - cross_kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2, None], - encoder_num_cached_tokens_per_seq: Optional[List[int]] = None, - ) -> "AttentionMetadata": - """Refresh an existing CUDA graph cross-attention sub-metadata.""" - if not self.has_cross_sub_metadata: - raise RuntimeError( - "CUDA graph cross-attention metadata has not been initialized.") - - cross_md = self.cross - assert cross_md is not None - base_kv_cache_params = cross_md.kv_cache_params - block_ids_per_seq = (base_kv_cache_params.block_ids_per_seq - if base_kv_cache_params is not None else None) - return self._update_cross_metadata( - cross_md, - encoder_seq_lens, - cross_kv_cache_manager, - encoder_num_cached_tokens_per_seq, - base_kv_cache_params=base_kv_cache_params, - block_ids_per_seq=block_ids_per_seq, - ) + """Build a cross-attention sub-metadata; see ``CrossAttentionMetadata``. - def update_for_spec_dec(self) -> None: - """ - Hook to be called during forward when using spec-dec one-model mode. + Attaches ``CrossAttentionMetadata`` on first use, so only models with + cross attention carry it. """ + from .features import CrossAttentionMetadata + return self.add_ext(CrossAttentionMetadata).create_sub_metadata( + encoder_seq_lens, cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq) @staticmethod def get_empty(buffers: Buffers, @@ -1179,6 +1125,32 @@ class AttentionBackend(Generic[TMetadata]): Base class for attention backends. """ Metadata: Type[TMetadata] = AttentionMetadata + # Extensions every metadata of this backend carries, e.g. a sparse + # algorithm's state; see extensions.py. + metadata_extensions: Tuple[Type[MetadataExtension], ...] = () + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + extensions = cls.__dict__.get("metadata_extensions") + carried = cls.Metadata.backend_extension_types + if extensions and not all( + any(issubclass(have, wanted) for have in carried) + for wanted in extensions): + # Metadata built through ``cls.Metadata`` carries the backend's + # extensions, as a subclass of the inherited metadata class. + base = cls.Metadata + cls.Metadata = type( + f"{cls.__name__}Metadata", (base, ), { + "__module__": + cls.__module__, + "backend_extension_types": + (*base.backend_extension_types, *extensions), + }) + + @classmethod + def create_metadata(cls, **kwargs) -> TMetadata: + """Build this backend's metadata with its extensions attached.""" + return cls.Metadata(**kwargs) def __init__( self, diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py index f92e85283c4f..d79087db23f3 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py @@ -24,7 +24,7 @@ from tensorrt_llm.quantization.mode import QuantAlgo from .indexer import CSA2Indexer -from .metadata import CSA2TrtllmMetadata +from .metadata import CSA2Metadata, CSA2TrtllmMetadata from .params import CSA2BackendForwardArgs, CSA2Layout, CSA2Mode, CSA2Params, select_csa2_backend if TYPE_CHECKING: @@ -81,7 +81,7 @@ class CSA2TrtllmAttention(TrtllmAttention): @classmethod def runtime_workspace_bytes_per_token(cls, model_config, mapping) -> int: - return CSA2TrtllmMetadata.cache_gather_bytes_per_token(model_config) + return CSA2Metadata.cache_gather_bytes_per_token(model_config) @classmethod def runtime_workspace_is_chunked_prefill_bounded(cls, model_config) -> bool: @@ -115,7 +115,7 @@ def runtime_workspace_fixed_bytes( ).multi_processor_count def arena_bytes(queries: int, page_capacity: int) -> int: - return CSA2TrtllmMetadata.workspace_reservation_bytes( + return CSA2Metadata.workspace_reservation_bytes( max_batch_size, queries, page_capacity * 64, @@ -288,7 +288,7 @@ def prepare_sparse_inputs( if ( q.shape != (count, self.num_heads * _HEAD_DIM) or q.dtype != torch.bfloat16 - or metadata.num_query_heads != self.num_heads + or metadata.ext(CSA2Metadata).num_query_heads != self.num_heads ): raise ValueError("CSA2 Q geometry/dtype must match its BF16 metadata") if forward_args.attention_input_type not in ( diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py index af5b9ed9a975..25c893657cff 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Iterator -from contextlib import contextmanager from copy import copy from dataclasses import replace from enum import Enum @@ -756,15 +755,24 @@ def _get_page_index_converter(self, layer_id, role): self._page_index_converters[key] = self.impl.get_page_index_converter(layer_id, role) return self._page_index_converters[key] - @contextmanager - def _swa_publication_scope(self): - token = object() + def prepare_batch_tables(self, request_ids, num_contexts) -> None: + # A publication opened by an earlier step's block-offset copy and never + # consumed must not reach this step's prepare. self._swa_publication = None - self._swa_publication_token = token - try: - yield token - finally: - # A nested prepare cannot resurrect an older publication either. + self._swa_publication_token = None + + def consume_swa_publication(self): + """The token of the SWA tables the latest block-offset copy published, or None. + + CSA2's prepare consumes it once, reads the published tables through + ``_get_swa_publication`` and then closes it with + ``close_swa_publication``. + """ + return self._swa_publication_token if self._swa_publication is not None else None + + def close_swa_publication(self, token) -> None: + """Close the publication ``token`` names; a later one stays open.""" + if token is not None and token is self._swa_publication_token: self._swa_publication = None self._swa_publication_token = None @@ -1027,6 +1035,7 @@ def copy_batch_block_offsets( ): # Invalidate before any validation/fill/copy can fail. self._swa_publication = None + self._swa_publication_token = None if beam_width != 1: raise ValueError("CSA2 supports beam width one") # Every per-layer table of the batch is converted at once; the SWA @@ -1039,14 +1048,20 @@ def copy_batch_block_offsets( # Context-only caches keep SWA for a prefix of the layers only. swa_layers = sum(self.has_swa_cache(layer) for layer in self.pp_layers) dst_tensor[:swa_layers, : len(request_ids), 0, :width].copy_(tables[:swa_layers, :, :width]) - if self._swa_publication_token is not None: - self._swa_publication = ( - dst_tensor, - tuple(request_ids), - num_contexts, - width, - (dst_tensor.data_ptr(), dst_tensor.shape, dst_tensor.stride()), - ) + # Publish the copied SWA tables for this step's CSA2 prepare. + self._swa_publication_token = object() + self._swa_publication = ( + dst_tensor, + tuple(request_ids), + num_contexts, + width, + (dst_tensor.data_ptr(), dst_tensor.shape, dst_tensor.stride()), + ) + + @property + def decoder_replay_window(self) -> int: + """Prompt tokens decoder bounded replay re-runs; 0 without decoder replay.""" + return self._decoder_replay_window @property def blocks_in_primary_pool(self): diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/decoder_replay.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/decoder_replay.py index 16c00b1a8a70..7301e9858c20 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/decoder_replay.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/decoder_replay.py @@ -41,12 +41,9 @@ @dataclass class DecoderReplayPlan: - """One forward's replay bookkeeping: what to run, and how to undo it. + """One forward's replay bookkeeping: what to run, and how to map back. - One record rather than loose locals because the "undo" half is not derivable - from the metadata once :func:`enter_decoder_replay` has overwritten it -- the - original ``seq_lens`` is the only place the encoder-pass row count survives, - and ``rows`` is the only map back to it. + ``rows`` is the only map from the replayed rows back to the encoder pass's. """ #: Global query rows, in the *encoder* pass's token space, that the decoder @@ -59,13 +56,7 @@ class DecoderReplayPlan: #: Per-request cached-token count for the replay pass, raised by exactly the #: number of query rows the pass drops. replay_num_cached: List[int] - #: ``seq_lens`` as the encoder pass saw it. Restoring this restores - #: ``num_tokens``, ``num_ctx_tokens`` and ``seq_lens_cuda`` with it, because - #: they are all derived through the ``seq_lens`` setter. - saved_seq_lens: torch.Tensor - saved_num_cached: List[int] replay_prompt_lens: List[int] - saved_prompt_lens: List[int] #: Row count of the encoder pass, i.e. the length every per-token activation #: crossing the boundary has on the way in and must have again on the way out. num_encoder_tokens: int @@ -190,13 +181,10 @@ def plan_decoder_replay( ), replay_seq_lens=torch.tensor(replay_lens, dtype=seq_lens.dtype), replay_num_cached=replay_num_cached, - saved_seq_lens=seq_lens, - saved_num_cached=num_cached, replay_prompt_lens=[ replay_lens[r] if r < num_contexts else int(metadata.prompt_lens[r]) for r in range(num_requests) ], - saved_prompt_lens=metadata.prompt_lens, num_encoder_tokens=metadata.num_tokens, swa_floors=floors, updates_local_metadata=updates_local_metadata, @@ -225,49 +213,24 @@ def enter_decoder_replay( metadata: "CSA2TrtllmMetadata", plan: DecoderReplayPlan, first_layer: Optional[int] = None, -) -> None: - """Rebuild query views and carry encoder publications across the boundary.""" - if plan.replay_all_rank_num_tokens is not None: - metadata.all_rank_num_tokens = plan.replay_all_rank_num_tokens +) -> "CSA2TrtllmMetadata": + """Return the metadata the decoder layers run on, with the encoder publications. + + A plan that compacts the local rows prepares the decoder replay metadata + of ``metadata``'s CSA2 (``prepare_decoder_replay``); ``metadata`` itself + is not changed. An attention-DP identity plan keeps ``metadata`` and only + presents the exchanged decoder token counts. + """ if not plan.updates_local_metadata: - return - saved_indices = dict(metadata.csa2_indices) - saved_candidates = { - name: dict(getattr(metadata, name)) - for name in ("csa2_candidates", "csa2_candidate_blocks", "csa2_candidate_counts") - } - saved_last_layer = metadata._csa2_last_layer - precomputed = set(getattr(metadata, "csa2_precomputed_kv_layers", ())) - live_kv_lens = metadata.kv_lens_cuda[: metadata.num_seqs].clone() - metadata.kv_cache_params.num_cached_tokens_per_seq[: len(plan.replay_num_cached)] = ( - plan.replay_num_cached - ) - metadata.prompt_lens = plan.replay_prompt_lens - metadata.seq_lens = plan.replay_seq_lens + if plan.replay_all_rank_num_tokens is not None: + metadata.all_rank_num_tokens = plan.replay_all_rank_num_tokens + return metadata + replay = metadata.prepare_decoder_replay(plan, first_layer) if plan.num_replay_tokens == 0: - # No local attention runs. Keep GLOBAL publication and live KV lengths, - # but do not ask attention backends to prepare an empty query batch. - metadata.reset_routing() - metadata.csa2_precomputed_kv_layers = precomputed - metadata.csa2_replay_query_rows = plan.rows - return - if first_layer is not None: - metadata.set_decoder_query_boundary(first_layer) - metadata.prepare() - metadata.kv_lens_cuda[: metadata.num_seqs].copy_(live_kv_lens) - metadata.on_update_kv_lens() - metadata.csa2_precomputed_kv_layers = precomputed - metadata.csa2_replay_query_rows = plan.rows - for layer, indices in saved_indices.items(): - metadata.csa2_indices[layer] = indices.index_select(0, plan.rows) - for name, published in saved_candidates.items(): - target = getattr(metadata, name) - for layer, value in published.items(): - target[layer] = value.index_select(0, plan.rows) - metadata._csa2_last_layer = saved_last_layer + return replay # Only context SWA is truncated. Generation rows and global visibility # retain their original logical positions and cache history. - starts = metadata._copy_csa2_tensor( + starts = replay._copy_csa2_tensor( "decoder_starts", torch.tensor( plan.swa_floors, @@ -275,7 +238,8 @@ def enter_decoder_replay( device="cpu", ), ) - _apply_decoder_swa_floor(metadata, starts) + _apply_decoder_swa_floor(replay, starts) + return replay def enter_remote_tail_decoder(metadata: "CSA2TrtllmMetadata", context_starts: List[int]) -> None: @@ -320,26 +284,17 @@ def exit_decoder_replay( metadata: "CSA2TrtllmMetadata", plan: DecoderReplayPlan, ) -> None: - """Restore the encoder pass's shape. + """Return ``metadata`` to the encoder pass's state after the decoder layers. - Only the *inputs* are restored -- ``seq_lens`` (and with it ``num_tokens``, - ``num_ctx_tokens`` and ``seq_lens_cuda``), the cached-token counts, and the - floor switch. The device index buffers are deliberately left holding replay - values: every forward rebuilds them from these inputs in ``prepare()``, and - copying them back would cost a second rebuild to no observable end. + An attention-DP identity plan restores its token counts. Otherwise the + decoder layers ran on the replay metadata and ``metadata`` kept its + lengths; its encoder publications, indexed in the encoder pass's rows, + are reset. """ - if plan.saved_all_rank_num_tokens is not None: + if plan.saved_all_rank_num_tokens is not None and not plan.updates_local_metadata: metadata.all_rank_num_tokens = plan.saved_all_rank_num_tokens if not plan.updates_local_metadata: return - metadata.kv_cache_params.num_cached_tokens_per_seq[: len(plan.saved_num_cached)] = ( - plan.saved_num_cached - ) - metadata.prompt_lens = plan.saved_prompt_lens - metadata.seq_lens = plan.saved_seq_lens - metadata.csa2_replay_query_rows = None - # Publications are indexed in the compact query space and cannot survive - # shape restoration. metadata.reset_routing() diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/indexer.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/indexer.py index 372d6c19cac5..4f3ed095041a 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/indexer.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/indexer.py @@ -1324,7 +1324,7 @@ def load_shared(begin=begin, end=end, width=width): continue if not is_prefill and paged is not None: continue - chunk_size = metadata.indexer_max_chunk_size + chunk_size = metadata.csa2_indexer_max_chunk_size for start in range(begin, end, chunk_size): stop = min(start + chunk_size, end) # Decode graphs must continue seeing newly published keys as diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py index 111314e2594c..dceae98febcd 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py @@ -388,7 +388,7 @@ def kernel( class OwnerSlotsKernel: """GLOBAL write slots and compressed groups of every KV owner for live endpoints. - Mirrors ``CSA2TrtllmMetadata._refresh_owner_slots`` in one launch: grid + Mirrors ``CSA2Metadata._refresh_owner_slots`` in one launch: grid ``y`` selects the owner, whose buffers and geometry come from an int64 ``[owners, OWNER_DESC_WIDTH]`` descriptor table (see ``owner_slot_descriptor``). Per-request group counts and their prefix @@ -1385,7 +1385,7 @@ def refresh_owner_slots( requests: int, max_capacity: int, ) -> None: - """Device implementation of ``CSA2TrtllmMetadata._refresh_owner_slots`` for + """Device implementation of ``CSA2Metadata._refresh_owner_slots`` for every owner of the int64 ``[owners, OWNER_DESC_WIDTH]`` ``descriptors``.""" ns = _glue_kernels() _check(requests <= ns.OwnerSlotsKernel.MAX_REQUESTS, "CSA2 owner refresh request capacity") @@ -1426,7 +1426,7 @@ def fill_indexer_descriptors( visible: torch.Tensor, valid: torch.Tensor, ) -> None: - """Device implementation of ``CSA2TrtllmMetadata._fill_indexer_descriptors``.""" + """Device implementation of ``CSA2Metadata._fill_indexer_descriptors``.""" count = int(blocks.shape[0]) if count == 0: return @@ -1885,7 +1885,7 @@ def map_logical_slot( max_positions: Int64, visible: Int64, ) -> Int64: - """``CSA2TrtllmMetadata._map_global_slots`` for one entry (visibility applied).""" + """``CSA2Metadata._map_global_slots`` for one entry (visibility applied).""" slot = Int64(-1) if (table_rows > 0) & (table_cols > 0) & (logical >= 0): page = logical // page_size @@ -4074,8 +4074,8 @@ def derive_stage_kv_scales( ) _launch( ("stage_scale", cache_format, mapping_signature, device.index), - lambda groups=scale_bytes, limit=_SCALE_BYTE_LIMITS[cache_format]: ( - ns.StageScaleKernel(groups, limit) + lambda groups=scale_bytes, limit=_SCALE_BYTE_LIMITS[cache_format]: ns.StageScaleKernel( + groups, limit ), _ptr(pool, align=1), _ptr(slots, align=8 if slots.dtype == torch.int64 else 4), @@ -4110,7 +4110,7 @@ def stage_selected_rows( swa_slots, main_pool, main_slots, - metadata, + tile, scratch, *, compact=True, @@ -4126,14 +4126,14 @@ def stage_selected_rows( Main selections are either physical ``main_slots`` or ``main_logical`` positions resolved in-kernel through ``main_mapping`` (``page_table``, ``page_size``, ``max_positions``, ``requests``, ``visible``), exactly as - ``CSA2TrtllmMetadata.global_slot_tile`` with visibility. ``scratch`` holds + ``CSA2Metadata.global_slot_tile`` with visibility. ``scratch`` holds the group's ``tags``/``counts`` (and, in the mapped form, ``main_slots`` receives the physical rows); ``compact=False`` reuses them from an earlier layer of the same group in this forward. """ - count = metadata.num_tokens - capacity = metadata.num_sparse_topk - fp8 = metadata.swa_pool.dtype == torch.float8_e4m3fn + count = tile.metadata.num_tokens + capacity = tile.metadata.num_sparse_topk + fp8 = tile.swa_pool.dtype == torch.float8_e4m3fn device = swa_pool.device has_main = main_slots is not None or main_logical is not None if not has_main: @@ -4151,12 +4151,12 @@ def stage_selected_rows( tags, counts = scratch["tags"], scratch["counts"] _check(capacity <= tags.shape[1] and capacity <= 4096, "staging capacity") ns = _cache_kernels() - placeholder = metadata.prepared_counter + placeholder = tile.prepared_counter dequant = kv_scale_quant_orig if fp8 else placeholder quant = kv_scale_orig_quant if fp8 else placeholder q_dequant = q_scale_quant_orig if fp8 else placeholder - bmm1 = metadata.mla_bmm1_scale if fp8 else placeholder - bmm2 = metadata.mla_bmm2_scale if fp8 else placeholder + bmm1 = tile.mla_bmm1_scale if fp8 else placeholder + bmm2 = tile.mla_bmm2_scale if fp8 else placeholder logical_int64 = logical.dtype == torch.int64 table_ptr, table_rows, table_cols, table_stride, page_size, max_positions = table_args if compact: @@ -4190,7 +4190,7 @@ def stage_selected_rows( int(capacity), _stream(device), ) - extra_rows = int(metadata.extra_pool.shape[1]) + extra_rows = int(tile.extra_pool.shape[1]) _launch( ("stage_dequant", fp8, device.index), lambda: ns.StageDequantKernel(fp8), @@ -4200,12 +4200,12 @@ def stage_selected_rows( _ptr(slots, align=8), _ptr(tags), _ptr(counts), - _ptr(metadata.swa_pool, align=8), - _ptr(metadata.extra_pool, align=8), + _ptr(tile.swa_pool, align=8), + _ptr(tile.extra_pool, align=8), _ptr(quant), - _ptr(metadata.prepared_lens), - _ptr(metadata.prepared_indices), - _ptr(metadata.prepared_counter), + _ptr(tile.prepared_lens), + _ptr(tile.prepared_indices), + _ptr(tile.prepared_counter), _ptr(q_dequant), _ptr(dequant), _ptr(bmm1), @@ -4235,7 +4235,7 @@ def stage_shared_rows( main_pool, main_slots, main_logical, - metadata, + tile, plan, *, main_mapping=None, @@ -4251,10 +4251,10 @@ def stage_shared_rows( their ``main_logical`` positions, or logical positions resolved through ``main_mapping``. Outputs preserve selection order and duplicates. """ - count = metadata.num_tokens - capacity = metadata.num_sparse_topk - bank = metadata.shared_pool - selected = metadata._csa2_shared_selected + count = tile.metadata.num_tokens + capacity = tile.metadata.num_sparse_topk + bank = tile.shared_pool + selected = tile._csa2_shared_selected device = swa_pool.device _check( bank.shape == (1 + plan.swa_rows + plan.main_rows, 512) and bank.is_contiguous(), @@ -4297,7 +4297,7 @@ def stage_shared_rows( selected.zero_() bank[0].zero_() ns = _cache_kernels() - placeholder = metadata.prepared_counter + placeholder = tile.prepared_counter q_dequant = q_scale_quant_orig if fp8 else placeholder _launch( ("shared_compact", fp8, has_main, premapped, device.index), @@ -4313,13 +4313,13 @@ def stage_shared_rows( _ptr(visible, align=8), table_ptr, _ptr(selected), - _ptr(metadata.prepared_indices), - _ptr(metadata.prepared_lens), - _ptr(metadata.prepared_counter), + _ptr(tile.prepared_indices), + _ptr(tile.prepared_lens), + _ptr(tile.prepared_counter), _ptr(q_dequant), _ptr(kv_scale_quant_orig if fp8 else placeholder), - _ptr(metadata.mla_bmm1_scale if fp8 else placeholder), - _ptr(metadata.mla_bmm2_scale if fp8 else placeholder), + _ptr(tile.mla_bmm1_scale if fp8 else placeholder), + _ptr(tile.mla_bmm2_scale if fp8 else placeholder), int(count), swa_width, main_width, diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py index 6e2742c1edad..8f411ecf004c 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py @@ -4,16 +4,15 @@ from __future__ import annotations -from contextlib import contextmanager, nullcontext -from dataclasses import dataclass +from dataclasses import dataclass, replace from math import prod from typing import TYPE_CHECKING import numpy as np import torch +from tensorrt_llm._torch.attention.backends.sparse.params import SparseAlgorithmExtension from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.metadata import KVCacheParams from .params import CSA2BackendForwardArgs, CSA2Layer @@ -66,7 +65,7 @@ class _CoalescedUploads: _ALIGN = 16 - def __init__(self, metadata: CSA2TrtllmMetadata): + def __init__(self, metadata: CSA2Metadata): self._metadata = metadata self._pending = [] self._bytes = 0 @@ -115,27 +114,322 @@ def flush(self) -> None: self._bytes = 0 -class CSA2TrtllmMetadata(TrtllmAttentionMetadata): +# A shared cache the source of a copy has not created yet. +_NOT_CREATED = object() + + +class CSA2Metadata(SparseAlgorithmExtension): """Manager-backed request metadata with bounded native query staging. Normal prepare resolves manager pages for the packed model forward. - for_query_tile creates a separate compute-only metadata object. + for_query_tile creates a separate compute-only metadata object that + carries its own CSA2 extension. Eager context retains real query groups with virtual staged KV lengths. Generation and captured context use independent one-query generation rows. Source causality, request isolation and windows are encoded in selections; these bounded compute pools are separate from persistent manager caches. """ - indexer_max_chunk_size: int = 8192 + # Caches every copy of the metadata must see: the query tiles, the + # shape-keyed buffer bank and staging, the indexer workspaces, resets and + # temporal priors, and the per-manager state of draft replays. + shared = frozenset( + { + "_csa2_query_tiles", + "_csa2_buffers", + "_csa2_eager_latest", + "_csa2_swa_table_cache", + "_csa2_host_staging", + "_csa2_upload_slab", + "_csa2_indexer_workspaces", + "_csa2_indexer_resets", + "_csa2_priors", + "csa2_indexer_prior_capacity", + } + ) + fields = frozenset( + { + "_csa2_buffers", + "_csa2_compressed_positions", + "_csa2_compression", + "_csa2_forward_serial", + "_csa2_graph_reserved", + "_csa2_indexer_workspaces", + "_csa2_last_layer", + "_csa2_main_domain_counts", + "_csa2_owner_descriptors", + "_csa2_query_base", + "_csa2_query_lengths", + "_csa2_query_offsets", + "_csa2_query_tiles", + "_csa2_shared_domains", + "_csa2_shared_selected", + "_csa2_source_geometry", + "_csa2_steady_snapshot", + "_csa2_swa_descriptors", + "_csa2_swa_page_tables", + "_csa2_swa_ratios", + "_csa2_swa_read_floors", + "_csa2_swa_resolved", + "adp_input_token_count", + "adp_token_counts", + "kv_cache_pools", + "csa2_candidate_blocks", + "csa2_candidate_counts", + "csa2_candidates", + "csa2_decode_row_requests", + "csa2_decoder_capture_lens", + "csa2_global_max_positions", + "csa2_global_page_sizes", + "csa2_global_page_tables", + "csa2_global_source_indices", + "csa2_indexer_block_table", + "csa2_indexer_context_lengths", + "csa2_indexer_k_cache", + "csa2_indexer_max_chunk_size", + "csa2_indexer_max_seq_len", + "csa2_indexer_prior_capacity", + "csa2_indexer_radix_aux_indices", + "csa2_indexer_radix_aux_logits", + "csa2_indexer_scheduler_metadata", + "csa2_indexer_valid_positions", + "csa2_indexer_visible_lengths", + "csa2_indices", + "csa2_kv_sources", + "csa2_main_write_slots", + "csa2_num_context_requests", + "csa2_positions", + "csa2_precomputed_kv_layers", + "csa2_query_layer_start", + "csa2_replay_cached_lengths", + "csa2_replay_mode", + "csa2_replay_query_rows", + "csa2_replay_signature", + "csa2_replay_start_positions", + "csa2_request_lengths", + "csa2_request_query_ranges", + "csa2_request_start_positions", + "csa2_swa_indices", + "csa2_swa_write_slots", + "csa2_token_requests", + "csa2_visible_lengths", + "builds_decoder_replay", + "decoder_context_ends", + "decoder_replay_metadata", + "decoder_replay_plan", + "extra_pool", + "indexer_q_split_threshold", + "latent_placeholder", + "mla_bmm1_scale", + "mla_bmm2_scale", + "native_unit_scale", + "pool_mapping", + "pool_pointers", + "prepared_counter", + "prepared_cu_kv", + "prepared_cu_q", + "prepared_indices", + "prepared_lens", + "query_lens_device", + "query_lens_host", + "csa2_remote_tail_mode", + "csa2_remote_tail_starts", + "shared_plan", + "shared_pool", + "stage_kv_scales", + "staging_dtype", + "swa_pool", + } + ) + init_fields = { + "builds_decoder_replay": None, + } + methods = frozenset( + { + "_allocate_tile", + "_bind_context_tile", + "_bind_swa_pages", + "_copy_csa2_tensor", + "_copy_host", + "_ensure_swa_slots", + "_get_csa2_buffer", + "_owner_slot_descriptors", + "_refresh_live_endpoints", + "_refresh_steady_page_tables", + "_shared_context_plan", + "_swa_ratio_tensor", + "_swa_table_descriptors", + "_workspace_storage_bytes", + "prepare_decoder_replay", + "receive_adp_token_counts", + "cache_gather_bytes_per_token", + "derive_stage_kv_scales", + "enter_layer", + "for_query_tile", + "get_compressed_positions", + "get_compression_batch", + "get_query_tile_metadata", + "get_workspace_bytes", + "global_slot_tile", + "prepare_context_replay", + "prepare_csa2", + "prepare_indexer", + "prepare_indexer_prior", + "prepare_sparse_indexer", + "publish_indexer_prior", + "register_indexer_reset", + "reset_forward_state", + "reset_routing", + "select_global_source", + "set_decoder_query_boundary", + "set_source_batch", + "set_swa_bounded_replay", + "stage_selected", + "workspace_reservation_bytes", + } + ) + + adopts_target_state = True + # A draft binds the target's prepared request metadata: request geometry, + # resolved page tables (including the SWA tables the target's prepare took + # from this step's publication), source selections and replay state. + draft_bound = frozenset( + { + "_csa2_forward_serial", + "_csa2_main_domain_counts", + "_csa2_missing_swa_slot", + "_csa2_query_base", + "_csa2_query_lengths", + "_csa2_query_offsets", + "_csa2_replay_capture_signature", + "_csa2_source_geometry", + "_csa2_stage_epoch", + "_csa2_steady_snapshot", + "_csa2_swa_descriptors", + "_csa2_swa_page_tables", + "_csa2_swa_ratios", + "_csa2_swa_read_floors", + "_csa2_swa_specs", + "csa2_decode_row_requests", + "csa2_global_max_positions", + "csa2_global_page_sizes", + "csa2_global_page_tables", + "csa2_global_source_indices", + "csa2_kv_sources", + "csa2_num_context_requests", + "csa2_query_layer_start", + "csa2_replay_cached_lengths", + "csa2_replay_mode", + "csa2_replay_signature", + "csa2_replay_start_positions", + "csa2_request_lengths", + "csa2_request_query_ranges", + "csa2_request_start_positions", + "csa2_token_requests", + "decoder_context_ends", + "csa2_remote_tail_mode", + "csa2_remote_tail_starts", + "sparse_metadata_params", + } + ) + # Rewritten in place by the live-endpoint refresh + # (_refresh_live_endpoints): query positions, owner write slots and compressed + # positions. The compression batches' endpoint tensors are written too; + # _adopt_target_request_state gives the draft its own. + draft_written = frozenset( + { + "_csa2_compressed_positions", + "csa2_main_write_slots", + "csa2_positions", + } + ) + # The draft's own buffer bank, host staging and descriptor tables, the + # compression batches and owner-slot descriptors over its own endpoint + # buffers, per-forward routing and SWA slots, pending prepare inputs, + # per-forward model state, indexer and staging scratch, temporal priors, + # and the bounded staging state of a compute tile. + draft_private = frozenset( + { + "_decoder_replay", + "builds_decoder_replay", + "_csa2_context_geometry", + "_csa2_adopted_compression", + "_csa2_buffers", + "_csa2_compression", + "_csa2_eager_latest", + "_csa2_graph_reserved", + "_csa2_host_staging", + "_csa2_indexer_resets", + "_csa2_indexer_workspaces", + "_csa2_last_layer", + "_csa2_owner_descriptors", + "_csa2_pending_query_layer_start", + "_csa2_pending_replay", + "_csa2_pending_swa_floors", + "_csa2_priors", + "_csa2_query_tiles", + "_csa2_shared_domains", + "_csa2_shared_selected", + "_csa2_source_batch", + "_csa2_stage_groups", + "_csa2_stage_kv_dequant", + "_csa2_stage_kv_quant", + "_csa2_stage_scale_bitmap", + "_csa2_swa_ratio_cache", + "_csa2_swa_resolved", + "_csa2_swa_table_cache", + "_csa2_upload_slab", + "csa2_candidate_blocks", + "csa2_candidate_counts", + "csa2_candidates", + "csa2_decoder_capture_lens", + "csa2_indexer_block_table", + "csa2_indexer_context_lengths", + "csa2_indexer_k_cache", + "csa2_indexer_max_seq_len", + "csa2_indexer_prior_capacity", + "csa2_indexer_radix_aux_indices", + "csa2_indexer_radix_aux_logits", + "csa2_indexer_scheduler_metadata", + "csa2_indexer_valid_positions", + "csa2_indexer_visible_lengths", + "csa2_indices", + "csa2_precomputed_kv_layers", + "csa2_replay_query_rows", + "csa2_swa_indices", + "csa2_swa_write_slots", + "csa2_visible_lengths", + "decoder_replay_plan", + "extra_pool", + "latent_placeholder", + "mla_bmm1_scale", + "mla_bmm2_scale", + "native_unit_scale", + "pool_mapping", + "pool_pointers", + "prepared_counter", + "prepared_cu_kv", + "prepared_cu_q", + "prepared_indices", + "prepared_lens", + "query_lens_device", + "query_lens_host", + "shared_plan", + "shared_pool", + "staging_dtype", + "swa_pool", + } + ) + + csa2_indexer_max_chunk_size: int = 8192 indexer_q_split_threshold: int = 8192 # Compute-tile metadata only: the shared converted-KV plan of an eager # context phase, or None for independent-query staging. shared_plan: CSA2SharedKVPlan | None = None - _csa2_defer_decode_outputs: bool = False - _csa2_deferred_decode_outputs: bool = False _csa2_graph_reserved: bool = False + # The manager's SWA publication token while ``prepare()`` runs. - # Request metadata is held directly on this object. Main/index physical + # Request metadata is held directly on this extension. Main/index physical # tables and write slots are owner-keyed; SWA and visibility are layer-keyed. csa2_indices: dict[int, torch.Tensor] csa2_candidates: dict[int, torch.Tensor] @@ -154,58 +448,111 @@ class CSA2TrtllmMetadata(TrtllmAttentionMetadata): csa2_global_max_positions: dict[int, int] csa2_kv_sources: dict[int, int | None] decoder_replay_plan: DecoderReplayPlan | None = None + # Whether this metadata builds the metadata its decoder replay runs on; + # None builds one for an eager target over a cache with decoder replay. + builds_decoder_replay: bool | None = None + _decoder_replay: TrtllmAttentionMetadata | None = None # Whole-prompt endpoints, frozen before overlap scheduling advances requests. # Unlike prompt_lens, these are not the current context chunk lengths. decoder_context_ends: tuple[int, ...] | None = None csa2_decoder_capture_lens: tuple[int, ...] | None = None + # Disaggregated remote-tail role of this step's context batch ("source", + # "destination") or None, and each context request's tail start. Derived + # from the step's context requests (``BatchInputs.context_requests``). + csa2_remote_tail_mode: str | None = None + csa2_remote_tail_starts: tuple[int, ...] = () - def get_adp_token_counts(self) -> list[int]: + @property + def adp_input_token_count(self) -> int: + """The input stage's local token count in the attention-DP exchange: + the padded count when the engine padded this step.""" + md = self.metadata + return md.padded_num_tokens or md.num_tokens + + @property + def adp_token_counts(self) -> list[int]: + """The decoder stage's local token count in the attention-DP exchange.""" plan = self.decoder_replay_plan - input_tokens = self.padded_num_tokens or self.num_tokens decoder_tokens = ( - plan.num_replay_tokens if plan and plan.replays_local_tokens else input_tokens + plan.num_replay_tokens + if plan and plan.replays_local_tokens + else self.adp_input_token_count ) - return [input_tokens, decoder_tokens] + return [decoder_tokens] - def set_adp_token_counts(self, counts: list[list[int]]) -> list[int]: + def receive_adp_token_counts(self, counts: list[list[int]], input_counts: list[int]) -> None: + """Take every rank's decoder-stage counts from the attention-DP exchange.""" plan = self.decoder_replay_plan if plan is not None: - plan.saved_all_rank_num_tokens, plan.replay_all_rank_num_tokens = counts - if not plan.updates_local_metadata and counts[0] == counts[1]: + plan.saved_all_rank_num_tokens, plan.replay_all_rank_num_tokens = ( + input_counts, + counts[0], + ) + if not plan.updates_local_metadata and input_counts == counts[0]: self.decoder_replay_plan = None - return counts[0] - def __post_init__(self) -> None: - # Graph clones rerun post-init: eligibility belongs to their own next + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Caches every copy shares with its source, created on first use: a + # copy takes each one its source already has. + for name in type(self).declared_shared(): + value = self.shared_or(name, lambda: _NOT_CREATED) + if value is not _NOT_CREATED: + setattr(self, name, value) + # Graph clones re-allocate: eligibility belongs to their own next # successful preparation, not the metadata object they were copied from. - self._csa2_deferred_decode_outputs = False self.decoder_replay_plan = None self.decoder_context_ends = None - super().__post_init__() + self._decoder_replay = self._build_decoder_replay() if self._replays_decoder() else None + + def _replays_decoder(self) -> bool: + """Whether this metadata needs a decoder replay metadata of its own.""" + if self.builds_decoder_replay is not None: + return self.builds_decoder_replay + md = self.metadata + # Replay runs only eagerly, over a cache that retains the decoder's + # replay window; a draft metadata is created with it off. + return not md.is_cuda_graph and bool( + getattr(md.kv_cache_manager, "decoder_replay_window", 0) + ) - @property - def tokens_per_block(self) -> int: - return ( - self.kv_cache_manager.tokens_per_block - if self.kv_cache_manager is not None - else _SWA_TILE + def _build_decoder_replay(self) -> TrtllmAttentionMetadata: + """The metadata decoder replay runs the decoder layers on. + + A copy of this metadata without the one-model draft and cross + attention, over the same KV cache, with buffers of its own but the + caches every copy shares (CSA2's buffer bank, priors, indexer state). Replay prepares it from + the replay plan's batch, so this metadata's lengths never change. + """ + from ...features import CrossAttentionMetadata, DraftMetadata + + md = self.metadata + return md.constructed_copy( + without=(DraftMetadata, CrossAttentionMetadata), + builds_decoder_replay=False, + # Buffers of its own, not views of this metadata's pooled ones. + cuda_graph_buffers=None, + seq_lens=None, + seq_lens_kv=None, ) @property - def host_kv_cache_pool_pointers(self) -> torch.Tensor: - return ( - self.pool_pointers - if hasattr(self, "pool_pointers") - else super().host_kv_cache_pool_pointers - ) + def decoder_replay_metadata(self) -> TrtllmAttentionMetadata: + """The metadata decoder replay runs the decoder layers on.""" + if self._decoder_replay is None: + raise RuntimeError("This metadata has no decoder replay metadata") + return self._decoder_replay @property - def host_kv_cache_pool_mapping(self) -> torch.Tensor: - return ( - self.pool_mapping - if hasattr(self, "pool_mapping") - else super().host_kv_cache_pool_mapping - ) + def kv_cache_pools(self) -> tuple[torch.Tensor, torch.Tensor, int] | None: + """Pools a compute tile attends over instead of the manager's, or None. + + Host pool pointers, host pool mapping and tokens per block of the + tile's bounded staging pools. + """ + if not hasattr(self, "pool_pointers"): + return None + return self.pool_pointers, self.pool_mapping, _SWA_TILE @property def stage_kv_scales(self) -> tuple[torch.Tensor, torch.Tensor]: @@ -217,7 +564,7 @@ def stage_kv_scales(self) -> tuple[torch.Tensor, torch.Tensor]: """ return self._csa2_stage_kv_quant, self._csa2_stage_kv_dequant - def _allocate( + def _allocate_tile( self, capacity: int, heads: int, @@ -226,6 +573,7 @@ def _allocate( staging_dtype: torch.dtype = torch.bfloat16, shared_rows: int | None = None, ) -> None: + md = self.metadata if staging_dtype not in (torch.bfloat16, torch.float8_e4m3fn): raise ValueError("CSA2 native staging requires BF16 or E4M3") self.staging_dtype = staging_dtype @@ -263,51 +611,43 @@ def _allocate( [[self.swa_pool.data_ptr(), 0]], dtype=torch.int64, device="cpu" ) self.pool_mapping = torch.zeros((1, 2), dtype=torch.int32, device="cpu") - self.num_sparse_topk = _SWA_TILE + extra_capacity - self.max_seq_len = self.num_sparse_topk - self.kv_cache_params = KVCacheParams(use_cache=True) - self.kv_cache_block_offsets = torch.zeros( - (1, capacity, 2, (self.num_sparse_topk + _SWA_TILE - 1) // _SWA_TILE), - dtype=torch.int32, - device=device, + kv_window = _SWA_TILE + extra_capacity + md.bind_fixed_kv_window( + kv_window, + torch.zeros( + (1, capacity, 2, (kv_window + _SWA_TILE - 1) // _SWA_TILE), + dtype=torch.int32, + device=device, + ), ) self.prepared_indices = torch.full( - (capacity, self.num_sparse_topk), -1, dtype=torch.int32, device=device + (capacity, md.num_sparse_topk), -1, dtype=torch.int32, device=device ) self.prepared_lens = torch.empty(capacity, dtype=torch.int32, device=device) self.prepared_counter = torch.zeros(1, dtype=torch.uint32, device=device) self.prepared_cu_q = torch.arange(capacity + 1, dtype=torch.int32, device=device) * heads self.prepared_cu_kv = ( - torch.arange(capacity + 1, dtype=torch.int32, device=device) * self.num_sparse_topk + torch.arange(capacity + 1, dtype=torch.int32, device=device) * md.num_sparse_topk ) self.query_lens_host = torch.ones(capacity, dtype=torch.int32, device="cpu") self.query_lens_device = torch.ones(capacity, dtype=torch.int32, device=device) - self.kv_lens.fill_(self.num_sparse_topk) - self.kv_lens_cuda.fill_(self.num_sparse_topk) - self.prompt_lens_cpu.fill_(1) - self.prompt_lens_cuda.fill_(1) - self.host_request_types.fill_(1) - self.host_total_kv_lens.zero_() - # Each query count has independent native workspace storage. - # Eager initialization and capture must use that same tensor. - self.cuda_graph_workspace = self.workspace - - @classmethod + + @staticmethod def for_query_tile( - cls, q: torch.Tensor, extra_width: int, *, staging_dtype=torch.bfloat16, context_lengths: list[int] | None = None, shared_rows: int | None = None, - ) -> CSA2TrtllmMetadata: + ) -> TrtllmAttentionMetadata: """Prepare fixed geometry before TrtllmAttention.forward allocates output. q is BF16 [queries, heads, 512]. Extra capacity is a selected-main-row bound, not the persistent cache size. Callers retain one metadata object per geometry and warm it with the normal forward before graph capture. As with TrtllmAttentionMetadata, set is_cuda_graph for captured calls. + The returned metadata carries the tile's staging in its CSA2Metadata. """ count, heads, dim = q.shape if context_lengths and ( @@ -323,28 +663,22 @@ def for_query_tile( with torch.cuda.device(q.device): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("Prepare and warm CSA2 metadata before CUDA Graph capture") - metadata = cls( + metadata = TrtllmAttentionMetadata( max_num_requests=len(context_lengths) if context_lengths else count, max_num_tokens=count, + extension_types=(CSA2Metadata,), ) - metadata._allocate( + tile = metadata.ext(CSA2Metadata) + metadata._allocate_tile( count, heads, (extra_width + 127) // 128 * 128, q.device, staging_dtype, shared_rows ) - metadata.num_query_heads = heads - metadata._seq_lens = metadata.query_lens_host - metadata._seq_lens_cuda = metadata.query_lens_device - metadata._num_contexts = metadata._num_ctx_tokens = 0 - metadata._num_generations = metadata._num_tokens = count - metadata._bind_runtime_views( - kv_lens_cuda=metadata.kv_lens_cuda, - kv_lens=metadata.kv_lens, - prompt_lens_cuda=metadata.prompt_lens_cuda, - prompt_lens_cpu=metadata.prompt_lens_cpu, - host_request_types=metadata.host_request_types, + tile.num_query_heads = heads + metadata.bind_generation_tile( + metadata.query_lens_host, + metadata.query_lens_device, + metadata.prepared_cu_q, + metadata.prepared_cu_kv, ) - metadata.host_total_kv_lens[1] = count * metadata.num_sparse_topk - metadata.cu_q_seqlens = metadata.prepared_cu_q - metadata.cu_kv_seqlens = metadata.prepared_cu_kv if context_lengths: metadata._bind_context_tile(context_lengths) return metadata @@ -352,14 +686,15 @@ def for_query_tile( def _shared_context_plan(self, q, extra_width, layer_idx): # Graph frames are filtered by the caller. Without logical provenance, # keep the existing selected-row path. + md = self.metadata if ( layer_idx is None or _SWA_TILE + (extra_width + 127) // 128 * 128 > 4096 or not hasattr(self, "_csa2_main_domain_counts") - or q.shape[0] != self.num_ctx_tokens + or q.shape[0] != md.num_ctx_tokens ): return None - manager = self.kv_cache_manager + manager = md.kv_cache_manager requests = self.csa2_num_context_requests lengths = self.csa2_request_lengths[:requests] if not lengths or sum(lengths) != q.shape[0]: @@ -437,12 +772,13 @@ def get_query_tile_metadata( *, staging_dtype=torch.bfloat16, layer_idx: int | None = None, - ) -> CSA2TrtllmMetadata: + ) -> TrtllmAttentionMetadata: """Share bounded staging, retaining real request groups for context.""" + md = self.metadata context_lengths = [] - if query_start is not None and query_start < self.num_ctx_tokens: + if query_start is not None and query_start < md.num_ctx_tokens: end = query_start + q.shape[0] - if end > self.num_ctx_tokens: + if end > md.num_ctx_tokens: raise ValueError("CSA2 query tiles must not cross the context/generation boundary") for begin, stop in self.csa2_request_query_ranges: lo, hi = max(begin, query_start), min(stop, end) @@ -455,7 +791,7 @@ def get_query_tile_metadata( # path; choose it during graph warmup too, so its workspace is ready. with torch.cuda.device(q.device): capturing = torch.cuda.is_current_stream_capturing() - if context_lengths and (self.is_cuda_graph or capturing): + if context_lengths and (md.is_cuda_graph or capturing): context_lengths = [] shared_plan = ( self._shared_context_plan(q, extra_width, layer_idx) @@ -507,24 +843,21 @@ def get_query_tile_metadata( shared_rows=shared_rows, ) if workspace is not None: - metadata.workspace = metadata.cuda_graph_workspace = workspace + metadata.use_workspace(workspace) self._csa2_query_tiles[key] = metadata - metadata.shared_plan = shared_plan - metadata._csa2_graph_reserved |= self.is_cuda_graph or capturing - metadata.is_cuda_graph = capturing + tile = metadata.ext(CSA2Metadata) + tile.shared_plan = shared_plan + metadata._csa2_graph_reserved |= md.is_cuda_graph or capturing + metadata.mark_cuda_graph(capturing) if capturing: self._csa2_replay_capture_signature = getattr(self, "csa2_replay_signature", None) if context_lengths: metadata._bind_context_tile(context_lengths) return metadata - def _bind_runtime_views(self, **kwargs) -> None: - # Generic prepare/rebinding can replace the geometry-only child views. - self._csa2_context_geometry = None - super()._bind_runtime_views(**kwargs) - def _bind_context_tile(self, query_lengths: list[int]) -> None: - geometry = (self.num_sparse_topk, tuple(query_lengths)) + md = self.metadata + geometry = (md.num_sparse_topk, tuple(query_lengths)) if getattr(self, "_csa2_context_geometry", None) == geometry: return # A failed partial update must not retain the previous successful key. @@ -533,27 +866,12 @@ def _bind_context_tile(self, query_lengths: list[int]) -> None: # context kernel still applies a causal upper bound in staged-column # coordinates: give even the first query the full sparse capacity, # otherwise short source prefixes would clip valid compressed keys. - kv_lengths = [self.num_sparse_topk + length - 1 for length in query_lengths] + kv_lengths = [md.num_sparse_topk + length - 1 for length in query_lengths] requests = len(query_lengths) query = torch.tensor(query_lengths, dtype=torch.int32, device="cpu") kv = torch.tensor(kv_lengths, dtype=torch.int32, device="cpu") self.query_lens_host[:requests].copy_(query) self._copy_host("context_query_lens", self.query_lens_device[:requests], query) - self._seq_lens = self.query_lens_host[:requests] - self._seq_lens_cuda = self.query_lens_device[:requests] - self._num_contexts = requests - self._num_ctx_tokens = self._num_tokens = sum(query_lengths) - self._num_generations = 0 - self.kv_lens[:requests].copy_(kv) - self._copy_host("context_kv_lens", self.kv_lens_cuda[:requests], kv) - # THOP interprets context_lengths as Q lengths, independently of - # past/current KV lengths. Keep them consistent with unfolded cuQ. - self.prompt_lens_cpu[:requests].copy_(query) - self._copy_host("context_prompt_lens", self.prompt_lens_cuda[:requests], query) - self.host_request_types[:requests].zero_() - self.host_total_kv_lens.zero_() - self.host_total_kv_lens[0] = sum(kv_lengths) - self.max_seq_len = max(self.num_sparse_topk, max(kv_lengths)) self._copy_host( "context_cu_q", self.prepared_cu_q[: requests + 1], @@ -564,14 +882,14 @@ def _bind_context_tile(self, query_lengths: list[int]) -> None: self.prepared_cu_kv[: requests + 1], torch.cat((kv.new_zeros(1), kv.cumsum(0).int())), ) - self.cu_q_seqlens = self.prepared_cu_q[: requests + 1] - self.cu_kv_seqlens = self.prepared_cu_kv[: requests + 1] - self._bind_runtime_views( - kv_lens_cuda=self.kv_lens_cuda[:requests], - kv_lens=self.kv_lens[:requests], - prompt_lens_cuda=self.prompt_lens_cuda[:requests], - prompt_lens_cpu=self.prompt_lens_cpu[:requests], - host_request_types=self.host_request_types[:requests], + md.bind_context_tile( + query, + kv, + query_lens_host=self.query_lens_host[:requests], + query_lens_device=self.query_lens_device[:requests], + q_offsets=self.prepared_cu_q[: requests + 1], + kv_offsets=self.prepared_cu_kv[: requests + 1], + copy_to_device=self._copy_host, ) self._csa2_context_geometry = geometry @@ -585,10 +903,15 @@ def _stage_scratch(self, source, group, count: int): compact. A capture records its own compaction so replays follow the live selections. """ + md, source_md = self.metadata, source.metadata groups = source.__dict__.setdefault("_csa2_stage_groups", {}) - key = (group, self.prepared_lens.shape[0], self.num_sparse_topk) + key = (group, self.prepared_lens.shape[0], md.num_sparse_topk) capturing = torch.cuda.is_current_stream_capturing() - frame = (id(source), id(source.kv_cache_manager), getattr(source, "_csa2_stage_epoch", 0)) + frame = ( + id(source), + id(source_md.kv_cache_manager), + getattr(source, "_csa2_stage_epoch", 0), + ) entry = groups.get(key) if entry is None: if capturing: @@ -602,18 +925,18 @@ def _stage_scratch(self, source, group, count: int): capacity, device = self.prepared_lens.shape[0], self.prepared_lens.device entry = groups[key] = { "tags": torch.empty( - (capacity, self.num_sparse_topk), dtype=torch.int32, device=device + (capacity, md.num_sparse_topk), dtype=torch.int32, device=device ), "counts": torch.empty(capacity, dtype=torch.int32, device=device), "main_slots": torch.empty( - (capacity, max(self.num_sparse_topk - _SWA_TILE, 1)), + (capacity, max(md.num_sparse_topk - _SWA_TILE, 1)), dtype=torch.int64, device=device, ), "serial": None, "graph_reserved": False, } - entry["graph_reserved"] |= self.is_cuda_graph or source.is_cuda_graph or capturing + entry["graph_reserved"] |= md.is_cuda_graph or source_md.is_cuda_graph or capturing serial = (*frame, count, capturing) compact = group is None or entry["serial"] != serial entry["serial"] = serial @@ -719,7 +1042,8 @@ def stage_selected( """ from .quantization import _fused_gather_supported, gather_rows, row_bytes - count = self.num_tokens + md = self.metadata + count = md.num_tokens fp8 = self.swa_pool.dtype == torch.float8_e4m3fn if fp8: kv_scale_orig_quant = ( @@ -759,7 +1083,7 @@ def stage_selected( if ( inputs.topk_indices.ndim != 2 or inputs.topk_indices.shape[0] != count - or inputs.topk_indices.shape[1] > self.num_sparse_topk - _SWA_TILE + or inputs.topk_indices.shape[1] > md.num_sparse_topk - _SWA_TILE ): raise ValueError("CSA2 selected main indices exceed metadata geometry") # Preserve gather_rows validation even for zero-width selections; an @@ -811,11 +1135,11 @@ def stage_selected( and all(pool.is_cuda and pool.stride(1) == 1 for pool, _, _ in sources) and _fused_gather_supported(inputs.swa_pool.device.index) ) - if native and self.num_sparse_topk <= 4096: + if native and md.num_sparse_topk <= 4096: from .kernel import stage_selected_rows - source = inputs.state.metadata if inputs.state is not None else None - scratch, compact = self._stage_scratch(source or self, inputs.stage_group, count) + source = inputs.state.metadata.ext(CSA2Metadata) if inputs.state is not None else self + scratch, compact = self._stage_scratch(source, inputs.stage_group, count) main_slots = inputs.topk_indices if mapping is not None and main_slots is not None: main_slots = scratch["main_slots"][:count, : main_slots.shape[1]] @@ -891,7 +1215,7 @@ def stage_selected( indices[:, :_SWA_TILE].copy_( torch.where(swa_positions < lengths[:, None], offsets * _SWA_TILE + swa_positions, -1) ) - extra_capacity = self.num_sparse_topk - _SWA_TILE + extra_capacity = md.num_sparse_topk - _SWA_TILE if extra_capacity: extra_positions = torch.arange(extra_capacity, device=swa.device)[None, :] indices[:, _SWA_TILE:].copy_( @@ -935,25 +1259,43 @@ def set_swa_bounded_replay( ) self._csa2_pending_replay = (tuple(cached_prefix_lengths), decoder) + def _bind_remote_tail(self, requests: list[LlmRequest]) -> None: + """Read the batch's remote-tail role and tail starts from its context requests.""" + modes = {getattr(request, "py_csa2_remote_tail_mode", None) for request in requests} + modes.discard(None) + if not modes: + self.csa2_remote_tail_mode = None + self.csa2_remote_tail_starts = () + return + if len(modes) != 1 or any( + getattr(request, "py_csa2_remote_tail_mode", None) is None for request in requests + ): + raise ValueError("CSA2 remote-tail context batches must be homogeneous") + self.csa2_remote_tail_mode = next(iter(modes)) + self.csa2_remote_tail_starts = tuple( + request.py_csa2_remote_tail_start for request in requests + ) + def prepare_context_replay( self, requests: list[LlmRequest], *, allow_final_window: bool = True ) -> None: """Resolve recovery sources and read floors before preparing model inputs.""" from .....pyexecutor.ced_replay import EncoderCheckpoint, EncoderReplay + md = self.metadata # Graph selection may vary between chunks. Keep every chunk's Decoder # history when a later chunk can enter a graph that expects full inputs. self.decoder_context_ends = ( tuple(request.prompt_len for request in requests) if allow_final_window else () ) - prefixes = [None] * self.num_seqs - floors = [0] * self.num_seqs - starts = self.kv_cache_params.num_cached_tokens_per_seq - lengths = self.seq_lens.tolist() + prefixes = [None] * md.num_seqs + floors = [0] * md.num_seqs + starts = md.kv_cache_params.num_cached_tokens_per_seq + lengths = md.seq_lens.tolist() for row, request in enumerate(requests): plan = request.py_ced_replay if isinstance(plan, (EncoderReplay, EncoderCheckpoint)): - cache = self.kv_cache_manager.kv_cache_map.get(request.py_request_id) + cache = md.kv_cache_manager.kv_cache_map.get(request.py_request_id) if cache is None or plan.cache_identity != id(cache) or not cache.is_active: raise ValueError("Encoder recovery has lost its active Global claim") if isinstance(plan, EncoderReplay): @@ -971,14 +1313,16 @@ def prepare_context_replay( self.set_swa_bounded_replay(prefixes) self._csa2_pending_swa_floors = floors - def set_decoder_query_boundary(self, first_layer: int) -> None: + def set_decoder_query_boundary(self, first_layer: int, ends: list[int]) -> None: """Prepare only decoder working pages after full GLOBAL publication. - Encoder pages before the current chunk may already be reclaimed. They - are not dependencies of this query-only pass and must not be reserved - again merely to build metadata for the longer decoder interval. + ``ends`` are the requests' query ends of the next prepare. Encoder + pages before the current chunk may already be reclaimed. They are not + dependencies of this query-only pass and must not be reserved again + merely to build metadata for the longer decoder interval. """ - layout = self.kv_cache_manager.layout + md = self.metadata + layout = md.kv_cache_manager.layout if ( first_layer not in self.csa2_precomputed_kv_layers or first_layer != max(layout.kv_source_layer_ids) @@ -989,13 +1333,7 @@ def set_decoder_query_boundary(self, first_layer: int) -> None: ) if hasattr(self, "_csa2_pending_replay") or hasattr(self, "_csa2_source_batch"): raise ValueError("Decoder query preparation cannot override pending source selection") - ends = [ - start + length - for start, length in zip( - self.kv_cache_params.num_cached_tokens_per_seq, self.seq_lens.tolist() - ) - ] - self.set_source_batch([0] * len(ends), ends) + self.set_source_batch([0] * len(ends), list(ends)) self._csa2_pending_query_layer_start = first_layer def select_global_source( @@ -1020,9 +1358,10 @@ def select_global_source( return hidden_states.index_select(0, indices) def _swa_replay_geometry(self, prefixes, decoder, starts, lengths) -> tuple: + md = self.metadata owner_shapes = [] - for owner in self.kv_cache_manager.layout.kv_source_layer_ids: - ratio = self.kv_cache_manager.layout.compress_ratios[owner] + for owner in md.kv_cache_manager.layout.kv_source_layer_ids: + ratio = md.kv_cache_manager.layout.compress_ratios[owner] source_lengths = tuple( length if prefix is None @@ -1034,27 +1373,17 @@ def _swa_replay_geometry(self, prefixes, decoder, starts, lengths) -> tuple: owner_shapes.append((owner, source_lengths, sum(source_lengths))) return ("decoder" if decoder else "encoder", tuple(lengths), tuple(owner_shapes)) - @contextmanager - def defer_cuda_graph_decode_outputs(self): - """Engine-only scope for a selected ordinary decode graph's preparation. - - The engine must run ``on_update_kv_lens`` before any model consumer, - including warmup and capture. Direct ``prepare`` callers retain fully - initialized outputs; graph ownership alone is not sufficient proof. - """ - previous = self._csa2_defer_decode_outputs - self._csa2_defer_decode_outputs = True - try: - yield - finally: - self._csa2_defer_decode_outputs = previous - def prepare(self) -> None: from .cache_manager import CSA2CacheManager - self._csa2_deferred_decode_outputs = False + md = self.metadata + if md.context_requests is not None: + self._bind_remote_tail(md.context_requests) + self.prepare_context_replay( + md.context_requests, allow_final_window=not md.prefill_cuda_graphs + ) self._csa2_context_geometry = None - if self.is_cuda_graph and hasattr(self, "_csa2_replay_capture_signature"): + if md.is_cuda_graph and hasattr(self, "_csa2_replay_capture_signature"): pending = getattr(self, "_csa2_pending_replay", None) signature = ( None @@ -1062,34 +1391,92 @@ def prepare(self) -> None: else self._swa_replay_geometry( pending[0], pending[1], - self.kv_cache_params.num_cached_tokens_per_seq, - self.seq_lens.tolist(), + md.kv_cache_params.num_cached_tokens_per_seq, + md.seq_lens.tolist(), ) ) if signature != self._csa2_replay_capture_signature: raise ValueError( "CSA2 replay mode/source geometry changed; use fresh graph metadata and recapture" ) - with torch.cuda.device(self.kv_lens_cuda.device): + with torch.cuda.device(md.kv_lens_cuda.device): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("Prepare CSA2 request metadata before CUDA Graph capture") - self._csa2_ready_for_kv_update = False - manager = self.kv_cache_manager - scope = ( - manager._swa_publication_scope() - if isinstance(manager, CSA2CacheManager) - else nullcontext(None) + manager = md.kv_cache_manager + # The core's block-offset copy published this step's SWA tables. + publication = ( + manager.consume_swa_publication() if isinstance(manager, CSA2CacheManager) else None ) - with scope as publication: - super().prepare() + try: self.prepare_csa2(swa_publication=publication) + finally: + if publication is not None: + manager.close_swa_publication(publication) + + def adopt_target_state(self, source: "CSA2Metadata") -> None: + """Start a draft from the target's prepared request metadata. + + The target's core opened this step's SWA publication and the target's + prepare consumed it; the draft binds the resolved tables and never + consumes one. The compression batches and owner-slot descriptors are + the draft's own, over its own endpoint buffers, and are rebuilt only + when the target prepared a new geometry; then only device copies run, + so a draft loop may take the target's state again during CUDA graph + capture. + """ + super().adopt_target_state(source) + compression = source._csa2_compression + adopted = getattr(self, "_csa2_adopted_compression", None) + if adopted is None or adopted[0] is not compression: + own = {} + for owner, batch in compression.items(): + previous = self._csa2_compression.get(owner) if adopted else None + endpoints = {} + for name in ("kv_lengths", "start_positions", "cu_compressed_lengths"): + value = getattr(batch, name) + mine = None if previous is None else getattr(previous, name) + endpoints[name] = ( + mine + if mine is not None and mine.shape == value.shape + else torch.empty_like(value) + ) + own[owner] = replace(batch, **endpoints) + self._csa2_compression = own + if not hasattr(self, "_csa2_buffers"): + self._csa2_buffers = {} + self._csa2_owner_descriptors = ( + None if source._csa2_owner_descriptors is None else self._owner_slot_descriptors() + ) + pairs = [ + (getattr(own[owner], name), getattr(batch, name)) + for owner, batch in compression.items() + for name in ("kv_lengths", "start_positions", "cu_compressed_lengths") + ] + adopted = self._csa2_adopted_compression = (compression, pairs) + if adopted[1]: + destinations, origins = zip(*adopted[1]) + torch._foreach_copy_(destinations, origins, non_blocking=True) + # The draft's forward resolves its own routing, SWA slots and context + # tile views. + self._csa2_context_geometry = None + self.reset_routing() + self.csa2_swa_indices = {} + self.csa2_swa_write_slots = {} + self.csa2_visible_lengths = {} + self._csa2_shared_domains = {} + self._csa2_swa_resolved = False + serial = self._csa2_forward_serial + for layer_idx, record in getattr(self, "_csa2_priors", {}).items(): + if record["serial"] != serial: + self._refresh_indexer_prior(layer_idx) def _get_csa2_buffer(self, key, shape, dtype, *, like=None) -> torch.Tensor: # Preserve manager-local, shape-keyed addresses across graph replays. + md = self.metadata if not hasattr(self, "_csa2_buffers"): self._csa2_buffers = {} - cache_key = (key, tuple(shape), dtype, self.is_cuda_graph) - if not self.is_cuda_graph: + cache_key = (key, tuple(shape), dtype, md.is_cuda_graph) + if not md.is_cuda_graph: # Eager buffers retain only the latest shape per name. Track that # shape per name so a prepare with many keys does not rescan the # whole bank for every lookup. @@ -1105,9 +1492,9 @@ def _get_csa2_buffer(self, key, shape, dtype, *, like=None) -> torch.Tensor: result = self._csa2_buffers.get(cache_key) if result is None: result = ( - torch.empty_like(like, device=self.kv_lens_cuda.device) + torch.empty_like(like, device=md.kv_lens_cuda.device) if like is not None - else torch.empty(shape, dtype=dtype, device=self.kv_lens_cuda.device) + else torch.empty(shape, dtype=dtype, device=md.kv_lens_cuda.device) ) self._csa2_buffers[cache_key] = result return result @@ -1161,11 +1548,12 @@ def _swa_write_pages(starts, lengths, floors, allocated, block, table_shape): "CSA2 query/replay interval requires writable SWA pages; " "reserve the complete replay range before prepare" ) - return CSA2TrtllmMetadata._flat_page_ranges(begin, np.where(active, end - begin, 0)) + return CSA2Metadata._flat_page_ranges(begin, np.where(active, end - begin, 0)) def _check_swa_write_pages(self, starts, lengths, floors, allocated_until, page_count): """Raise unless every SWA page the query intervals write is mapped.""" - manager = self.kv_cache_manager + md = self.metadata + manager = md.kv_cache_manager write_rows, write_columns = self._swa_write_pages( starts, lengths, @@ -1185,7 +1573,8 @@ def _check_owner_pages(self, owner, owner_starts, owner_ends, allocated_until, p inside the allocated capacity are mapped.""" from .cache_manager import CSA2CacheRole - manager = self.kv_cache_manager + md = self.metadata + manager = md.kv_cache_manager block = manager.tokens_per_block ratio = manager.layout.compress_ratios[owner] page_size = block // ratio @@ -1259,6 +1648,7 @@ def _swa_table_descriptors(self, layers: int) -> torch.Tensor: def _device_descriptors(self, kind: str, rows) -> torch.Tensor: """Device int64 table of the host descriptor ``rows``, uploaded only when they change.""" + md = self.metadata key = (kind, tuple(rows)) cache = getattr(self, "_csa2_swa_table_cache", None) if cache is None or cache[0] is not self._csa2_buffers: @@ -1266,7 +1656,7 @@ def _device_descriptors(self, kind: str, rows) -> torch.Tensor: # descriptor index with it so every copy sees the same fills. cache = self._csa2_swa_table_cache = (self._csa2_buffers, {}, {}) _, names, filled = cache - if self.is_cuda_graph: + if md.is_cuda_graph: # A captured launch reads its descriptor table at replay time, so # the content must never change after the fill: every distinct # binding (one per captured batch shape) owns its own buffer. @@ -1296,11 +1686,12 @@ def _ensure_swa_slots(self) -> None: re-arm it, so a graph capture always records the launch. """ # None: not prepared by prepare_csa2. + md = self.metadata if getattr(self, "_csa2_swa_resolved", None) is not False: return from . import kernel - manager = self.kv_cache_manager + manager = md.kv_cache_manager positions, requests = self.csa2_positions, self.csa2_token_requests layers, count, window = ( # SWA layers are a prefix; a decoder-only pass binds its suffix. @@ -1410,13 +1801,14 @@ def _refresh_swa_tensor_outputs( def _bind_swa_pages(self, layer, pages, published_pages): # Select backing once for each existing shape/graph key. A direct or # draft fallback must update it, never redirect a captured consumer. + md = self.metadata if published_pages is not None and ( published_pages.dtype != pages.dtype - or published_pages.device != self.kv_lens_cuda.device + or published_pages.device != md.kv_lens_cuda.device or published_pages.shape != pages.shape ): published_pages = None - key = (f"swa_pages/{layer}", tuple(pages.shape), pages.dtype, self.is_cuda_graph) + key = (f"swa_pages/{layer}", tuple(pages.shape), pages.dtype, md.is_cuda_graph) if not hasattr(self, "_csa2_buffers"): self._csa2_buffers = {} if published_pages is not None and key not in self._csa2_buffers: @@ -1443,27 +1835,26 @@ def prepare_csa2(self, *, swa_publication=None) -> None: from .cache_manager import CSA2CacheManager, CSA2CacheRole from .compressor import CSA2CompressionBatch - self._csa2_deferred_decode_outputs = False - self._csa2_ready_for_kv_update = False + md = self.metadata # Every prepared forward starts a new staging epoch: the first layer of # each staging group compacts again. self._csa2_stage_epoch = getattr(self, "_csa2_stage_epoch", 0) + 1 - manager = self.kv_cache_manager + manager = md.kv_cache_manager if not isinstance(manager, CSA2CacheManager): raise TypeError("CSA2 runtime metadata requires CSA2CacheManager") self.csa2_query_layer_start = getattr(self, "_csa2_pending_query_layer_start", 0) if hasattr(self, "_csa2_pending_query_layer_start"): del self._csa2_pending_query_layer_start - if self.beam_width != 1 or self.is_spec_dec_dynamic_tree: + if md.beam_width != 1 or md.is_spec_dec_dynamic_tree: raise NotImplementedError( "CSA2 supports contiguous-prefix verification, not beams or dynamic trees" ) - request_ids = list(self.request_ids) - lengths = self.seq_lens.tolist() - if manager.context_swa_layer_limit is not None and self.num_contexts != len(lengths): + request_ids = list(md.request_ids) + lengths = md.seq_lens.tolist() + if manager.context_swa_layer_limit is not None and md.num_contexts != len(lengths): raise ValueError("CSA2 context-only cache cannot prepare generation requests") - manager.validate_verification(lengths, self.num_contexts, self.is_spec_decoding_enabled) - starts = list(self.kv_cache_params.num_cached_tokens_per_seq) + manager.validate_verification(lengths, md.num_contexts, md.is_spec_decoding_enabled) + starts = list(md.kv_cache_params.num_cached_tokens_per_seq) if len(request_ids) != len(lengths) or len(starts) != len(lengths): raise ValueError("CSA2 requires one query length and cached length per request") replay = getattr(self, "_csa2_pending_replay", None) @@ -1489,7 +1880,7 @@ def prepare_csa2(self, *, swa_publication=None) -> None: for row, (prefix, start, length) in enumerate(zip(prefixes, starts, lengths)): if prefix is None: continue - if row >= self.num_contexts: + if row >= md.num_contexts: raise ValueError("CSA2 bounded replay applies only to context requests") if start != replay_starts[row]: raise ValueError( @@ -1516,20 +1907,6 @@ def prepare_csa2(self, *, swa_publication=None) -> None: ) ) external_source = hasattr(self, "_csa2_source_batch") - # Only the engine's selected ordinary decode graph may leave derived - # outputs for its captured preprocessing. Source overrides, replay and - # speculative/draft metadata keep the complete preparation contract. - defer_outputs = ( - getattr(self, "_csa2_defer_decode_outputs", False) - and self.is_cuda_graph - and self.num_contexts == 0 - and bool(lengths) - and all(length == 1 for length in lengths) - and not self.is_spec_decoding_enabled - and replay is None - and not external_source - and getattr(self, "csa2_remote_tail_mode", None) is None - ) source_lengths, source_starts = getattr(self, "_csa2_source_batch", (lengths, starts)) if external_source: del self._csa2_source_batch @@ -1537,11 +1914,12 @@ def prepare_csa2(self, *, swa_publication=None) -> None: raise ValueError("CSA2 source and query batches must name the same requests") # Under the overlap scheduler, generation rows reserve the previous # step's full verification width; positions beyond the allocated - # capacity stay unmapped and on_update_kv_lens narrows them later. + # capacity stay unmapped until the corrected device KV lengths narrow + # them (_refresh_live_endpoints). allocated_until = np.full(len(lengths), manager.max_seq_len, dtype=np.int64) - allocated_until[self.num_contexts :] = [ + allocated_until[md.num_contexts :] = [ manager.kv_cache_map[request_id].capacity - for request_id in request_ids[self.num_contexts :] + for request_id in request_ids[md.num_contexts :] ] lengths_np = np.asarray(lengths, dtype=np.int64) starts_np = np.asarray(starts, dtype=np.int64) @@ -1567,9 +1945,8 @@ def prepare_csa2(self, *, swa_publication=None) -> None: ) self.csa2_request_start_positions = tuple(starts) self.reset_routing() - self._finish_prepare_csa2(snapshot, defer_outputs=defer_outputs) - if not defer_outputs: - self._refresh_live_endpoints() + self._finish_prepare_csa2(snapshot) + self._refresh_live_endpoints() return self._csa2_steady_snapshot = None ends = np.asarray(source_starts, dtype=np.int64) + np.asarray( @@ -1583,7 +1960,7 @@ def prepare_csa2(self, *, swa_publication=None) -> None: raise ValueError("CSA2 query positions exceed the manager context capacity") self.csa2_request_start_positions = tuple(starts) self.csa2_request_lengths = tuple(lengths) - self.csa2_num_context_requests = self.num_contexts + self.csa2_num_context_requests = md.num_contexts query_ends = np.cumsum(lengths_np) query_begins = query_ends - lengths_np packed_start = int(query_ends[-1]) if query_ends.size else 0 @@ -1596,11 +1973,7 @@ def prepare_csa2(self, *, swa_publication=None) -> None: query_begins, lengths_np ) positions_np = starts_np[token_requests_np] + query_offsets_np - self.csa2_positions = ( - self._get_csa2_buffer("positions", (packed_start,), torch.int32) - if defer_outputs - else copy("positions", positions_np.astype(np.int32)) - ) + self.csa2_positions = copy("positions", positions_np.astype(np.int32)) self.csa2_replay_start_positions = copy( "replay_starts", np.asarray(replay_starts, dtype=np.int64) ) @@ -1622,10 +1995,10 @@ def prepare_csa2(self, *, swa_publication=None) -> None: self._csa2_compression = {} self._csa2_compressed_positions = {} published_swa = manager._get_swa_publication( - swa_publication, self.kv_cache_block_offsets, request_ids, self.num_contexts, page_count + swa_publication, md.kv_cache_block_offsets, request_ids, md.num_contexts, page_count ) if published_swa is None: - manager.compute_batch_page_tables(request_ids, self.num_contexts) + manager.compute_batch_page_tables(request_ids, md.num_contexts) # Owner slots are resolved from these host KV ends: a draft prepare # runs without the generic prepare, so kv_lens_cuda may hold live # lengths that differ from the host positions. @@ -1638,7 +2011,7 @@ def prepare_csa2(self, *, swa_publication=None) -> None: ) shared_cu = copy("source_cu", np.cumsum([0, *source_lengths], dtype=np.int32)) owners = manager.layout.kv_source_layer_ids - if self.num_contexts: + if md.num_contexts: # Reserve every currently addressable logical row, including # upward live-end corrections within already published pages. # This is capacity only; selected indices retain visibility. @@ -1687,7 +2060,7 @@ def prepare_csa2(self, *, swa_publication=None) -> None: ) capacity = sum(owner_lengths) global_spec = (owner, CSA2CacheRole.GLOBAL) - if self.num_contexts: + if md.num_contexts: self._csa2_main_domain_counts[owner] = tuple( min(page * (block // ratio), manager.max_seq_len // ratio) for page in last_pages[index] @@ -1752,11 +2125,11 @@ def prepare_csa2(self, *, swa_publication=None) -> None: self.csa2_token_requests = copy("token_requests", token_requests_np) # Generation-row -> generation-request map for the sparse paged indexer kernels decode_start = ( - query_ranges[self.num_contexts][0] - if self.num_contexts < len(query_ranges) + query_ranges[md.num_contexts][0] + if md.num_contexts < len(query_ranges) else packed_start ) - decode_rows_np = (token_requests_np[decode_start:] - self.num_contexts).astype(np.int32) + decode_rows_np = (token_requests_np[decode_start:] - md.num_contexts).astype(np.int32) self.csa2_decode_row_requests = ( copy("decode_row_requests", decode_rows_np) if decode_rows_np.size else None ) @@ -1797,8 +2170,9 @@ def prepare_csa2(self, *, swa_publication=None) -> None: if self.csa2_positions.is_cuda and kernel.dsl_available() else None ) - if not defer_outputs: - # A deferred decode graph resolves them in on_update_kv_lens. + if md.kv_lens_offsets is None: + # Otherwise the live-endpoint refresh at the end of prepare + # resolves them from the corrected device KV lengths. self._resolve_owner_slots(prepared_ends) self._csa2_swa_descriptors = self._csa2_swa_ratios = None if self.csa2_positions.is_cuda and kernel.dsl_available() and swa_layers: @@ -1812,13 +2186,17 @@ def prepare_csa2(self, *, swa_publication=None) -> None: # Only a plain generation prepare seeds the steady path: context, # replay, source-override and decoder-boundary geometry must be rebuilt. steady_seed = ( - not self.num_contexts + not md.num_contexts and replay is None and not external_source and not self.csa2_query_layer_start - and getattr(self, "csa2_remote_tail_mode", None) is None + and self.csa2_remote_tail_mode is None ) - self._finish_prepare_csa2(snapshot if steady_seed else None, defer_outputs=defer_outputs) + self._finish_prepare_csa2(snapshot if steady_seed else None) + if md.kv_lens_offsets is not None: + # The core corrected the device KV lengths these host positions + # do not carry. + self._refresh_live_endpoints() def _steady_generation_step(self, snapshot) -> bool: """Whether ``snapshot`` continues the previous prepare by one committed step. @@ -1826,8 +2204,9 @@ def _steady_generation_step(self, snapshot) -> bool: The same generation requests, in the same order and with the same query lengths, each start where the previous query ended. """ + md = self.metadata previous = getattr(self, "_csa2_steady_snapshot", None) - if previous is None or self.num_contexts: + if previous is None or md.num_contexts: return False ids, lengths, starts, _ = snapshot if (ids, lengths) != (previous[0], previous[1]): @@ -1837,9 +2216,10 @@ def _steady_generation_step(self, snapshot) -> bool: def _steady_pages_moved(self, snapshot) -> bool: """Whether a steady step's last position, SWA window start or allocated capacity crossed a page boundary, so its page tables may have changed.""" + md = self.metadata previous = self._csa2_steady_snapshot - block = self.kv_cache_manager.tokens_per_block - window = self.kv_cache_manager.layout.window_size + block = md.kv_cache_manager.tokens_per_block + window = md.kv_cache_manager.layout.window_size _, lengths, starts, allocated = snapshot last = starts + np.asarray(lengths, dtype=np.int64) - 1 prev_last = last - (starts - previous[2]) @@ -1862,12 +2242,13 @@ def _refresh_steady_page_tables( """ from .cache_manager import CSA2CacheRole - manager = self.kv_cache_manager + md = self.metadata + manager = md.kv_cache_manager published_swa = manager._get_swa_publication( - swa_publication, self.kv_cache_block_offsets, request_ids, self.num_contexts, page_count + swa_publication, md.kv_cache_block_offsets, request_ids, md.num_contexts, page_count ) if published_swa is None: - manager.compute_batch_page_tables(request_ids, self.num_contexts) + manager.compute_batch_page_tables(request_ids, md.num_contexts) self._check_swa_write_pages(starts, lengths, floors, allocated_until, page_count) ends = [start + length for start, length in zip(starts, lengths)] for owner in self.csa2_global_page_tables: @@ -1894,107 +2275,103 @@ def refresh(target, spec, published=None): if targets: torch._foreach_copy_(targets, sources) - def _finish_prepare_csa2(self, snapshot, *, defer_outputs: bool) -> None: - manager = self.kv_cache_manager - self._csa2_prepared_manager = manager + def _finish_prepare_csa2(self, snapshot) -> None: self._csa2_forward_serial = getattr(self, "_csa2_forward_serial", 0) + 1 for layer_idx in getattr(self, "_csa2_priors", {}): self._refresh_indexer_prior(layer_idx) self._csa2_steady_snapshot = snapshot - self._csa2_ready_for_kv_update = True - self._csa2_deferred_decode_outputs = bool(defer_outputs) - - def _cache_dependent_fields(self) -> dict: - shared = {"_csa2_query_tiles", "_csa2_indexer_workspaces", "_csa2_manager_states"} - return { - name: value - for name, value in vars(self).items() - if (name.startswith("csa2_") or name.startswith("_csa2_")) and name not in shared - } - - def _restore_cache_fields(self, fields: dict) -> None: - for name in self._cache_dependent_fields(): - delattr(self, name) - for name, value in fields.items(): - setattr(self, name, value) - def prepare_for_draft_forward(self) -> dict | None: - """Rebuild draft-owned fields after the existing interface swaps managers. - - Only identical explicit layer layouts are representable here. Virtual - model-layer mappings and CED draft scheduling belong to model integration. - This hook runs before capture/replay, never inside a captured forward. + def prepare_decoder_replay( + self, plan: DecoderReplayPlan, first_layer: int | None + ) -> TrtllmAttentionMetadata: + """Prepare the decoder replay metadata for ``plan`` and return it. + + Called on the target's CSA2 at the decoder boundary of its forward. + The replay metadata is prepared from the plan's batch (the replayed + query rows, with the dropped rows counted as cached) and the target's + KV-length correction, so its KV lengths equal the target's; the + encoder half's publications are carried over in the replayed rows' + order. The target metadata is left as it is. """ - target = getattr(self, "_csa2_prepared_manager", None) - draft = self.kv_cache_manager - if target is None or target is draft: - return None - if torch.cuda.is_current_stream_capturing(): - raise RuntimeError("Prepare CSA2 draft cache metadata before graph capture") - if getattr(draft, "layout", None) != target.layout: - raise NotImplementedError( - "CSA2 draft cache requires an identical explicit layer mapping" + from ...interface import BatchInputs + + md = self.metadata + replay = self.decoder_replay_metadata + replay_csa2 = replay.ext(CSA2Metadata) + num_requests = len(plan.replay_num_cached) + cached = list(md.kv_cache_params.num_cached_tokens_per_seq) + cached[:num_requests] = plan.replay_num_cached + inputs = BatchInputs( + seq_lens=plan.replay_seq_lens, + num_contexts=md.num_contexts, + request_ids=md.request_ids, + prompt_lens=plan.replay_prompt_lens, + kv_cache_params=replace(md.kv_cache_params, num_cached_tokens_per_seq=cached), + beam_width=md.beam_width, + num_chunked_ctx_requests=md.num_chunked_ctx_requests, + kv_lens_offsets=md.kv_lens_offsets, + ) + # The attention-DP ranks' decoder token counts, exchanged for replay. + all_rank_num_tokens = ( + md.all_rank_num_tokens + if plan.replay_all_rank_num_tokens is None + else plan.replay_all_rank_num_tokens + ) + replay.csa2_precomputed_kv_layers = set(self.csa2_precomputed_kv_layers) + if plan.num_replay_tokens == 0: + # No local attention runs: the replay metadata presents the empty + # query batch without preparing attention for it, and keeps the + # GLOBAL publication. + replay.core().apply_batch_inputs(inputs) + replay.all_rank_num_tokens = all_rank_num_tokens + replay.reset_routing() + replay.csa2_replay_query_rows = plan.rows + return replay + if first_layer is not None: + replay.set_decoder_query_boundary( + first_layer, + [start + length for start, length in zip(cached, plan.replay_seq_lens.tolist())], ) - saved = self._cache_dependent_fields() - if not hasattr(self, "_csa2_manager_states"): - self._csa2_manager_states = {} - self._restore_cache_fields(self._csa2_manager_states.get(id(draft), {})) - try: - self.prepare_csa2() - # Even contiguous draft requests may follow target execution in - # the same TopK module. Manager transitions always invalidate its - # emission state, independently of the restored request identities. - callbacks = list(saved.get("_csa2_indexer_resets", {}).values()) - callbacks += list(getattr(self, "_csa2_indexer_resets", {}).values()) - for callback in callbacks: - callback() - except (ValueError, TypeError, KeyError, NotImplementedError): - self._restore_cache_fields(saved) - raise - return saved - - def restore_after_draft_forward(self, saved_state: dict | None) -> None: - if saved_state is None: - return - draft = self._csa2_prepared_manager - self._csa2_manager_states[id(draft)] = self._cache_dependent_fields() - self._restore_cache_fields(saved_state) - # Target and draft may use the same module's emission buffers, while - # their request histories remain independent. Reset before target reuse. - for callback in getattr(self, "_csa2_indexer_resets", {}).values(): - callback() - - def begin_model_forward(self) -> None: + replay.prepare(inputs) + replay.all_rank_num_tokens = all_rank_num_tokens + replay.csa2_precomputed_kv_layers = set(self.csa2_precomputed_kv_layers) + replay.csa2_replay_query_rows = plan.rows + for layer, indices in self.csa2_indices.items(): + replay.csa2_indices[layer] = indices.index_select(0, plan.rows) + for name in ("csa2_candidates", "csa2_candidate_blocks", "csa2_candidate_counts"): + published = getattr(replay_csa2, name) + for layer, value in getattr(self, name).items(): + published[layer] = value.index_select(0, plan.rows) + replay._csa2_last_layer = self._csa2_last_layer + return replay + + def reset_forward_state(self) -> None: """Reset only per-model-forward publications, retaining prepared buffers.""" self.reset_routing() self.csa2_precomputed_kv_layers = set() self.csa2_replay_query_rows = None self.csa2_decoder_capture_lens = None - def on_update_kv_lens(self) -> None: - """Refresh query/cache addresses after device-side acceptance correction. - - Host cached lengths reserve verification capacity; live device endpoints - may be shorter. All address/ratio computations stay on device and retain - the prepared tensor shapes for captured replay. - """ - super().on_update_kv_lens() - if getattr(self, "_csa2_ready_for_kv_update", False): - self._refresh_live_endpoints() + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + # A draft step keeps the prepared query and cache addresses; only the + # previous forward's publications are reset. + self.reset_forward_state() def _refresh_live_endpoints(self) -> None: """Recompute positions and owner slots; SWA slots follow in the next forward.""" + md = self.metadata self._csa2_shared_domains = {} self._csa2_swa_resolved = False _, _, positions = self._refresh_query_positions( - self.kv_lens_cuda, + md.kv_lens_cuda, self._csa2_query_lengths, self._csa2_query_base, self.csa2_token_requests.long(), self._csa2_query_offsets, ) self.csa2_positions.copy_(positions) - self._resolve_owner_slots(self.kv_lens_cuda) + self._resolve_owner_slots(md.kv_lens_cuda) def _resolve_owner_slots(self, kv_lens: torch.Tensor) -> None: """Every KV owner's GLOBAL write slots and compressed groups for the KV ends @@ -2034,7 +2411,8 @@ def _resolve_owner_slots(self, kv_lens: torch.Tensor) -> None: def _owner_slot_arguments(self): """``(owner, (source base, source lengths, pages, slots, compressed, ratio, page size, batch starts, batch lengths, batch cu))`` of every KV owner.""" - manager = self.kv_cache_manager + md = self.metadata + manager = md.kv_cache_manager for owner, (source_base, source_lengths) in self._csa2_source_geometry.items(): ratio = manager.layout.compress_ratios[owner] batch = self._csa2_compression.get(owner) if ratio == 2 else None @@ -2190,7 +2568,7 @@ def _map_global_slots(logical, requests, table, visible_lengths, page_size, max_ slots = torch.where(logical < visible_lengths[:, None], slots, -1) return slots - def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: + def prepare_indexer(self, layer_idx: int) -> CSA2Metadata | None: """Bind native paged-indexer descriptors for the real generation queries. The owner's index rows are read in place: a GLOBAL page is a run of @@ -2206,10 +2584,11 @@ def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: from .quantization import INDEX_PAGE_ROWS - manager = self.kv_cache_manager + md = self.metadata + manager = md.kv_cache_manager if manager is None: raise ValueError("CSA2 paged indexer requires a cache manager") - if torch.cuda.is_current_stream_capturing() and not self.is_cuda_graph: + if torch.cuda.is_current_stream_capturing() and not md.is_cuda_graph: raise RuntimeError("Set is_cuda_graph before warming CSA2 paged indexer metadata") owner = self.csa2_kv_sources[layer_idx] if owner is None: @@ -2226,7 +2605,7 @@ def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: count = decode_end - decode_start request_count = len(generation_ranges) ratio = manager.layout.compress_ratios[owner] - if self.is_cuda_graph: + if md.is_cuda_graph: max_positions = max(1, self.csa2_global_max_positions[owner]) else: max_positions = max( @@ -2240,7 +2619,7 @@ def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: ), ) native_page_size = INDEX_PAGE_ROWS - if self.is_cuda_graph: + if md.is_cuda_graph: # Reserve the largest owner geometry during warmup so another # serialized owner cannot replace a buffer captured by this graph. required_pages = max( @@ -2255,17 +2634,17 @@ def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: table = self.csa2_global_page_tables[owner][context_requests:] device = table.device draft_width = 1 + getattr(manager, "max_total_draft_tokens", 0) - query_capacity = self.max_num_requests * draft_width if self.is_cuda_graph else count - if self.is_cuda_graph and count > query_capacity: + query_capacity = md.max_num_requests * draft_width if md.is_cuda_graph else count + if md.is_cuda_graph and count > query_capacity: raise ValueError("CSA2 graph query count exceeds configured batch and draft width") key = ( (device, manager.layout.index_topk, True) - if self.is_cuda_graph + if md.is_cuda_graph else (device, request_count, count, manager.layout.index_topk, False) ) if not hasattr(self, "_csa2_indexer_workspaces"): self._csa2_indexer_workspaces = {} - if not self.is_cuda_graph: + if not md.is_cuda_graph: # Runtime metadata survives many batch geometries. Retain just # the latest eager arena per device; active descriptors still own # their tensors, and serialized stream work preserves their use. @@ -2327,7 +2706,7 @@ def prepare_indexer(self, layer_idx: int) -> CSA2TrtllmMetadata | None: id(manager), getattr(self, "_csa2_forward_serial", 0), count, - self.is_cuda_graph, + md.is_cuda_graph, torch.cuda.is_current_stream_capturing(), ) if scratch["owners"].get(owner) != serial: @@ -2383,6 +2762,7 @@ def prepare_sparse_indexer( """ from tensorrt_llm.deep_gemm import get_paged_sparse_mqa_logits_metadata + md = self.metadata row_requests = getattr(self, "csa2_decode_row_requests", None) table = self.csa2_indexer_block_table owner = self.csa2_kv_sources[layer_idx] @@ -2402,10 +2782,10 @@ def prepare_sparse_indexer( counts = counts[counts.shape[0] - count :].reshape(-1) blocks = blocks[blocks.shape[0] - count :] serial = ( - id(self.kv_cache_manager), + id(md.kv_cache_manager), getattr(self, "_csa2_forward_serial", 0), count, - self.is_cuda_graph, + md.is_cuda_graph, torch.cuda.is_current_stream_capturing(), ) schedules = self.__dict__.setdefault("_csa2_sparse_schedules", {}) @@ -2570,24 +2950,24 @@ def visit(value): def get_workspace_bytes(self) -> int: """Report retained GPU workspace once per storage, excluding KV pools.""" + md = self.metadata values = [ value - for name, value in vars(self).items() + for name, value in (*vars(md).items(), *vars(self).items()) if isinstance(value, torch.Tensor) and name != "position_ids" ] + values += [md.workspace, md.cuda_graph_workspace] for name in ( - "workspace", - "cuda_graph_workspace", "_csa2_buffers", "_csa2_indexer_workspaces", "_csa2_priors", - "_csa2_manager_states", "_csa2_shared_domains", ): values.append(getattr(self, name, None)) for metadata in getattr(self, "_csa2_query_tiles", {}).values(): - for name, value in vars(metadata).items(): - if name != "kv_cache_manager": + tile = metadata.ext(CSA2Metadata) + for name, value in (*vars(metadata).items(), *vars(tile).items()): + if name not in ("kv_cache_manager", "metadata"): values.append(value) return self._workspace_storage_bytes(values) @@ -2598,6 +2978,7 @@ def register_indexer_reset(self, layer_idx: int, callback) -> None: self._csa2_indexer_resets[layer_idx] = callback def prepare_indexer_prior(self, layer_idx: int, topk: int) -> torch.Tensor: + md = self.metadata if not hasattr(self, "_csa2_priors"): self._csa2_priors = {} self.csa2_indexer_prior_capacity = {} @@ -2605,7 +2986,7 @@ def prepare_indexer_prior(self, layer_idx: int, topk: int) -> torch.Tensor: if record is None: if torch.cuda.is_current_stream_capturing(): raise RuntimeError("Warm up CSA2 temporal priors before graph capture") - capacity = self.max_num_tokens + capacity = md.max_num_tokens device = self.csa2_token_requests.device record = { "prior": torch.full((capacity, topk), -1, dtype=torch.int32, device=device), @@ -2625,17 +3006,18 @@ def prepare_indexer_prior(self, layer_idx: int, topk: int) -> torch.Tensor: return record["prior"][: record["count"]] def _refresh_indexer_prior(self, layer_idx: int) -> None: + md = self.metadata record = self._csa2_priors[layer_idx] previous = {key: row for row, key in enumerate(record["keys"]) if key is not None} rows, identities = [], [] publish_keys = [None] * sum(self.csa2_request_lengths) reset = self.csa2_num_context_requests > 0 for request_row, length in enumerate(self.csa2_request_lengths): - request_id = self.request_ids[request_row] + request_id = md.request_ids[request_row] start = self.csa2_request_start_positions[request_row] identity = ( - id(self.kv_cache_manager), - self.kv_cache_manager.request_epoch(request_id), + id(md.kv_cache_manager), + md.kv_cache_manager.request_epoch(request_id), request_id, ) begin, end = self.csa2_request_query_ranges[request_row] @@ -2685,3 +3067,9 @@ def get_compression_batch(self, owner: int): def get_compressed_positions(self, owner: int) -> torch.Tensor: return self._csa2_compressed_positions[owner] + + +class CSA2TrtllmMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the CSA2 extension attached.""" + + backend_extension_types = (CSA2Metadata,) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/__init__.py index a822bd90bd40..aa0a6e01f864 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/__init__.py @@ -16,7 +16,7 @@ from .backend import DeepseekV4TrtllmAttention from .cache_manager import DeepseekV4CacheManager from .indexer import DeepseekV4Indexer -from .metadata import DeepseekV4TrtllmAttentionMetadata +from .metadata import DeepseekV4Metadata, DeepseekV4TrtllmAttentionMetadata from .params import DeepseekV4AttentionType, DeepSeekV4MetadataParams, DeepSeekV4Params __all__ = [ @@ -26,5 +26,6 @@ "DeepseekV4CacheManager", "DeepseekV4Indexer", "DeepseekV4TrtllmAttention", + "DeepseekV4Metadata", "DeepseekV4TrtllmAttentionMetadata", ] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/cache_manager.py index b6246695d2df..b8c366d16f6d 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/cache_manager.py @@ -1680,6 +1680,12 @@ def _copy_scratch_metadata_to_device( self._device_scratch_slots_staging, ) + def prepare_batch_tables(self, request_ids: List[int], num_contexts: int) -> None: + # Publish an offloaded batch first; copy_batch_block_offsets then copies + # the SWA tables computed here. + super().prepare_batch_tables(request_ids, num_contexts) + self.compute_sliding_block_tables(request_ids, num_contexts) + @nvtx_range_debug("dsv4_compute_sliding_block_tables") def compute_sliding_block_tables( self, diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/metadata.py index 98477cd5fff9..4c85b62ed5cf 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/metadata.py @@ -13,7 +13,7 @@ from tensorrt_llm._torch.utils import maybe_compile from tensorrt_llm._utils import prefer_pinned -from ..dsa.metadata import DSAtrtllmAttentionMetadata +from ..dsa.metadata import DSAMetadata from ..offload import ( SparseOffloadState, allocate_sparse_offload_state, @@ -32,7 +32,126 @@ from .cache_manager import DeepseekV4CacheManager -class DeepseekV4TrtllmAttentionMetadata(DSAtrtllmAttentionMetadata): +class DeepseekV4Metadata(DSAMetadata): + """DeepSeek-V4 compressed and sliding-window state on top of the DSA indexer state.""" + + adopts_target_state = True + # A draft binds the target's configuration, host lengths, block tables, + # cache pointers and compression counts. + draft_bound = frozenset( + { + "_compress_ratios_sorted", + "_ctx_output_sizes", + "_mixed_precision_high_table_layers", + "attention_type_set", + "cached_token_lens_cpu", + "cached_token_lens_cuda", + "compress_block_tables", + "compress_ratio_set", + "compressed_buffer_ptrs", + "compressed_mask", + "cu_seq_lens", + "empty_topk_indices_buffer", + "max_compressed_indices", + "max_ctx_compressed_tokens", + "num_gen_tokens_per_seq", + "num_total_compressed_tokens", + "sliding_block_tables", + "sparse_mla_base_ptrs", + "sparse_offload_state", + "swa_buffer_ptrs", + "window_size", + } + ) + # Rewritten in place by a draft step's length refresh (refresh_from_draft_lengths) + # and by the fused Q prefixes of its forwards. + draft_written = frozenset( + { + "batch_size_cuda", + "compressed_kv_lens_cuda", + "compressed_local_indices_cuda", + "compressed_mask_cuda", + "compressed_position_ids_cuda", + "cu_new_comp_kv_cuda", + "cu_seq_lens_cuda", + "fused_q_gen_cu_seqlens", + "gen_new_tokens_per_seq_cuda", + "gen_position_params_cuda", + "new_comp_kv_lens_cuda", + "past_kv_lens_cuda", + "sparse_mla_extra_lens", + "sparse_mla_swa_lens", + "sparse_mla_topk_lens", + "swa_local_indices_cuda", + } + ) + # Scratch and caches its forwards build lazily, and the view of its own + # per-request token counts (rebound on adoption). + draft_private = frozenset( + { + "_fused_q_gen_cu_seqlens_valid", + "builds_mixed_precision_high_tables", + "_footer_scale_pools", + "gen_new_tokens_per_seq", + "nvfp4_compress_fp8_scratch", + } + ) + init_fields = { + # Whether the mixed-precision high page tables are built; a draft + # metadata's MTP layers have no mixed-precision high layer. + "builds_mixed_precision_high_tables": True, + } + + fields = frozenset( + { + "attention_type_set", + "builds_mixed_precision_high_tables", + "batch_size_cuda", + "cached_token_lens_cpu", + "cached_token_lens_cuda", + "compress_block_tables", + "compress_ratio_set", + "compressed_buffer_ptrs", + "compressed_kv_lens_cuda", + "compressed_local_indices_cuda", + "compressed_mask", + "compressed_mask_cuda", + "compressed_position_ids_cuda", + "cu_new_comp_kv_cuda", + "cu_seq_lens", + "cu_seq_lens_cuda", + "empty_topk_indices_buffer", + "_footer_scale_pools", + "fused_q_gen_cu_seqlens", + "gen_new_tokens_per_seq", + "gen_new_tokens_per_seq_cuda", + "max_compressed_indices", + "max_ctx_compressed_tokens", + "new_comp_kv_lens_cuda", + "num_gen_tokens_per_seq", + "num_total_compressed_tokens", + "nvfp4_compress_fp8_scratch", + "past_kv_lens_cuda", + "sliding_block_tables", + "sparse_mla_base_ptrs", + "sparse_mla_extra_lens", + "sparse_mla_swa_lens", + "sparse_mla_topk_lens", + "swa_buffer_ptrs", + "swa_local_indices_cuda", + "window_size", + } + ) + methods = frozenset( + { + "mla_prepare_fused_q_gen_cu_seqlens", + "prepare_compressed_kv_metadata", + "prepare_for_block_tables", + "prepare_for_deepseek_v4_indices", + "validate_sparse_offload_batch", + } + ) + # The set of compress ratios for the layers compress_ratio_set: Set[int] # The set of (compress ratio, attention type) for the layers @@ -41,21 +160,39 @@ class DeepseekV4TrtllmAttentionMetadata(DSAtrtllmAttentionMetadata): num_total_compressed_tokens: Dict[int, int] # The max number of context compressed tokens for each compress ratio max_ctx_compressed_tokens: Dict[int, int] + # Pool-base-relative footer-scale views, keyed by + # (id(kv_cache_manager), attention type, compress ratio, page size). + _footer_scale_pools: Optional[Dict[tuple, torch.Tensor]] = None - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def _invalidate_mla_scheduler_buffers(self) -> None: - """Invalidate request- and token-prefix buffers derived from sequence lengths.""" - super()._invalidate_mla_scheduler_buffers() + def _invalidate_length_prefixes(self) -> None: + super()._invalidate_length_prefixes() self._fused_q_gen_cu_seqlens_valid = False - def __post_init__(self): - super().__post_init__() + @property + def has_mixed_precision_high_rows(self) -> bool: + """Whether this backend writes and reads a mixed-precision high pool. + + A draft metadata's MTP layers have none, also when the draft takes the + target's page tables. + """ + return ( + self.mixed_precision_high_block_table is not None + and self.builds_mixed_precision_high_tables + ) + + def _allocate_mixed_precision_high_block_table(self, capture_graph: bool) -> None: + # One table per compression ratio, allocated with the compressed block tables. + pass + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + md = self.metadata self.num_total_compressed_tokens = {} self.max_ctx_compressed_tokens = {} self.nvfp4_compress_fp8_scratch: Optional[torch.Tensor] = None self._ctx_output_sizes: Optional[Dict[int, int]] = None + if md.kv_cache_manager is None and self.sparse_metadata_params is None: + return sparse_metadata_params = self.sparse_metadata_params if not isinstance(sparse_metadata_params, DeepSeekV4MetadataParams): raise ValueError("DeepSeek-V4 sparse attention metadata params are not set") @@ -65,7 +202,7 @@ def __post_init__(self): f"Dual-pool sparse MLA requires window_size == 128, which equals to the" f"TileSizeKV of the FMHA kernel. (got {window_size})." ) - capture_graph = self.is_cuda_graph + capture_graph = md.is_cuda_graph self.compress_ratio_set = set(self.compress_ratios) # Cache a sorted list for deterministic iteration order across # torch.compile traces. Avoids repeated set→list conversion and @@ -102,9 +239,9 @@ def __post_init__(self): # Create buffers for the compressor # cu_seq_lens_cuda is the cumulative sequence lengths for the requests - self.cu_seq_lens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), + self.cu_seq_lens_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences + 1,), dtype=torch.int, cache_name="cu_seq_lens_cuda", capture_graph=capture_graph, @@ -121,18 +258,18 @@ def __post_init__(self): if self.enable_ragged_verification: # The fused Q normalization op consumes request-major token # prefixes, rather than the sparse-MLA token-major row view. - self.fused_q_gen_cu_seqlens = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), + self.fused_q_gen_cu_seqlens = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences + 1,), cache_name="fused_q_gen_cu_seqlens", dtype=torch.int32, capture_graph=capture_graph, ) self.fused_q_gen_cu_seqlens[:1].zero_() # Exact tokens appended by each generation request. - self.gen_new_tokens_per_seq_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + self.gen_new_tokens_per_seq_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), dtype=torch.int, cache_name="gen_new_tokens_per_seq_cuda", capture_graph=capture_graph, @@ -140,9 +277,9 @@ def __post_init__(self): # new_comp_kv_lens_cuda is the number of new compressed tokens for the requests self.new_comp_kv_lens_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), dtype=torch.int, cache_name=f"new_comp_kv_lens_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -152,9 +289,9 @@ def __post_init__(self): # cu_new_comp_kv_cuda is the cumulative number of new compressed tokens for the requests self.cu_new_comp_kv_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences + 1,), dtype=torch.int, cache_name=f"cu_new_comp_kv_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -164,9 +301,9 @@ def __post_init__(self): # compressed_kv_lens_cuda is the number of compressed tokens for the requests self.compressed_kv_lens_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), dtype=torch.int, cache_name=f"compressed_kv_lens_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -176,9 +313,9 @@ def __post_init__(self): # past_kv_lens_cuda is the number of past compressed tokens for the requests self.past_kv_lens_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), dtype=torch.int, cache_name=f"past_kv_lens_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -188,9 +325,9 @@ def __post_init__(self): # compressed_position_ids_cuda is the compressed position ids for the requests self.compressed_position_ids_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), dtype=torch.int, cache_name=f"compressed_position_ids_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -199,8 +336,8 @@ def __post_init__(self): } # Device scalar batch size for the shape-static compressed_mask graph. - self.batch_size_cuda = self.get_empty( - self.cuda_graph_buffers, + self.batch_size_cuda = md.get_empty( + md.cuda_graph_buffers, (1,), dtype=torch.int, cache_name="batch_size_cuda", @@ -209,8 +346,8 @@ def __post_init__(self): # Device scalars (num_contexts, num_generations, num_gen_tokens_per_seq) # for the shape-static generation position-id graph. - self.gen_position_params_cuda = self.get_empty( - self.cuda_graph_buffers, + self.gen_position_params_cuda = md.get_empty( + md.cuda_graph_buffers, (3,), dtype=torch.int, cache_name="gen_position_params_cuda", @@ -220,9 +357,9 @@ def __post_init__(self): # compressed_mask_cuda: per-token bool mask for postprocess scatter. # Precomputed from new_comp_kv_lens to skip padded generation slots. self.compressed_mask_cuda = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), dtype=torch.bool, cache_name=f"compressed_mask_cuda_{compress_ratio}", capture_graph=capture_graph, @@ -237,9 +374,9 @@ def __post_init__(self): } # empty topk indices buffer with all -1s in the tensor - self.empty_topk_indices_buffer = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens, self.sparse_mla_topk), + self.empty_topk_indices_buffer = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens, self.sparse_mla_topk), cache_name="empty_topk_indices_buffer", dtype=torch.int32, capture_graph=capture_graph, @@ -247,9 +384,9 @@ def __post_init__(self): self.empty_topk_indices_buffer.fill_(-1) # SWA local indices - self.swa_local_indices_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens, self.window_size), + self.swa_local_indices_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens, self.window_size), cache_name="swa_local_indices_cuda", dtype=torch.int32, capture_graph=capture_graph, @@ -259,9 +396,9 @@ def __post_init__(self): # lengths and the fused output RoPE). The DSA base allocates them only # for the fp8_ds_mla layout, so provide them for every DSv4 cache mode. if getattr(self, "token_positions_cuda", None) is None: - self.token_positions_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.token_positions_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="token_positions_cuda", dtype=torch.int32, capture_graph=capture_graph, @@ -270,7 +407,7 @@ def __post_init__(self): # Compute max_compressed_indices for CUDA graph compatibility. # For ratio=4, the indexer selects index_topk compressed tokens. # For ratio=128, use max_seq_len / 128 rounded up to next power of 2 - raw_128 = math.ceil(self.max_seq_len / 128) + raw_128 = math.ceil(md.max_seq_len / 128) po2_128 = 1 << (raw_128 - 1).bit_length() if raw_128 > 0 else 1 self.max_compressed_indices = { 1: 0, # No compressed indices @@ -280,9 +417,9 @@ def __post_init__(self): # Compressed local indices for compress_ratio=128 # Note: ratio=4 uses dynamic topk_indices from indexer, so we only pre-allocate for ratio=128 - self.compressed_local_indices_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens, self.max_compressed_indices[128]), + self.compressed_local_indices_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens, self.max_compressed_indices[128]), cache_name="compressed_local_indices_cuda", dtype=torch.int32, capture_graph=capture_graph, @@ -293,13 +430,13 @@ def __post_init__(self): # uses ratio-shared page indices, and INDEXER_COMPRESS keeps a separate # compatibility table for the generic DSA indexer path. block_table_shape = ( - self.kv_cache_manager.num_local_layers, + md.kv_cache_manager.num_local_layers, len(DEEPSEEK_V4_SLIDING_ATTENTION), - self.max_num_sequences, - self.kv_cache_manager.max_blocks_per_seq, + md.max_num_sequences, + md.kv_cache_manager.max_blocks_per_seq, ) - self.sliding_block_tables = self.get_empty( - self.cuda_graph_buffers, + self.sliding_block_tables = md.get_empty( + md.cuda_graph_buffers, block_table_shape, cache_name="sliding_block_tables", dtype=torch.int32, @@ -307,12 +444,12 @@ def __post_init__(self): ) compress_block_table_shape = ( - self.max_num_sequences, - self.kv_cache_manager.max_blocks_per_seq, + md.max_num_sequences, + md.kv_cache_manager.max_blocks_per_seq, ) self.compress_block_tables = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, + compress_ratio: md.get_empty( + md.cuda_graph_buffers, compress_block_table_shape, cache_name=f"compress_block_tables_{compress_ratio}", dtype=torch.int32, @@ -322,15 +459,16 @@ def __post_init__(self): if is_compress_layer(compress_ratio) } # One mixed-precision high page table per compression ratio; -1 = not in the high pool. - manager = self.kv_cache_manager - if manager.mixed_precision_high_layers: - self._mixed_precision_high_table_layers = {} + # The MTP draft layers of a draft metadata have no mixed-precision high layer. + manager = md.kv_cache_manager + self._mixed_precision_high_table_layers: Dict[int, int] = {} + if manager.mixed_precision_high_layers and self.builds_mixed_precision_high_tables: for local, high_layer in manager.mixed_precision_high_layers.items(): ratio = manager._compress_ratios[manager.pp_layers[local]] self._mixed_precision_high_table_layers.setdefault(ratio, high_layer) self.mixed_precision_high_block_table = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, + compress_ratio: md.get_empty( + md.cuda_graph_buffers, compress_block_table_shape, cache_name=f"mixed_precision_high_block_table_{compress_ratio}", dtype=torch.int32, @@ -343,9 +481,9 @@ def __post_init__(self): # sparse_mla_topk_lens: actual token count per token for each compress_ratio (SWA + compressed) # Shape: [max_num_tokens] per compress_ratio self.sparse_mla_topk_lens = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name=f"sparse_mla_topk_lens_{compress_ratio}", dtype=torch.int32, capture_graph=capture_graph, @@ -356,17 +494,17 @@ def __post_init__(self): # per step next to sparse_mla_topk_lens: SWA rows live per token # (min(pos + 1, window)) and compressed rows live per compress ratio # (sparse_mla_topk_lens[ratio] - window). Shape: [max_num_tokens]. - self.sparse_mla_swa_lens = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.sparse_mla_swa_lens = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="sparse_mla_swa_lens", dtype=torch.int32, capture_graph=capture_graph, ) self.sparse_mla_extra_lens = { - compress_ratio: self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + compress_ratio: md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name=f"sparse_mla_extra_lens_{compress_ratio}", dtype=torch.int32, capture_graph=capture_graph, @@ -376,9 +514,9 @@ def __post_init__(self): } # cached_token_lens_cuda: number of tokens already cached per request - self.cached_token_lens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + self.cached_token_lens_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), cache_name="cached_token_lens_cuda", dtype=torch.int32, capture_graph=capture_graph, @@ -391,29 +529,27 @@ def __post_init__(self): # so compute them once during initialization instead of every prepare(). self._init_cache_buffer_data_pointers() - # Draft-sized sparse buffers for one-model MTP separate draft KV cache. - self._init_draft_sparse_buffers() - def _init_sparse_offload_state(self) -> None: """Validate fixed offload configuration and allocate persistent metadata.""" + md = self.metadata self.sparse_offload_state: SparseOffloadState | None = None - if not self.sparse_metadata_params.enable_kv_cache_offload: + if not md.sparse_metadata_params.enable_kv_cache_offload: return - manager = self.kv_cache_manager + manager = md.kv_cache_manager # Descriptors include every sparse role in native fetch order. - layers = get_sparse_offload_metadata_layers(self) + layers = get_sparse_offload_metadata_layers(md) if not layers: # This PP stage has no ratio-4 main KV to stage. return # Native fetch always processes every Batch row, including for smaller # graph profiles. Its selection width is fixed by BufferConfig. self.sparse_offload_state = allocate_sparse_offload_state( - lambda name, shape, dtype: self.get_empty( - self.cuda_graph_buffers, + lambda name, shape, dtype: md.get_empty( + md.cuda_graph_buffers, shape, dtype=dtype, cache_name=f"dsv4_sparse_offload_{name}", - capture_graph=self.is_cuda_graph, + capture_graph=md.is_cuda_graph, ), layers, max_batch_size=manager.max_batch_size, @@ -431,13 +567,14 @@ def mla_prepare_fused_q_gen_cu_seqlens(self) -> Optional[torch.Tensor]: into graph-owned storage once per iteration; device-window graph replay therefore updates values without changing the captured address. """ - if not self.is_ragged_verify or self.num_generations <= 0: + md = self.metadata + if not self.is_ragged_verify or md.num_generations <= 0: return None - num_contexts = self.num_contexts - num_generations = self.num_generations + num_contexts = md.num_contexts + num_generations = md.num_generations cu_seqlens = getattr(self, "fused_q_gen_cu_seqlens", None) - seq_lens_cuda = getattr(self, "seq_lens_cuda", None) + seq_lens_cuda = getattr(md, "seq_lens_cuda", None) if cu_seqlens is None or seq_lens_cuda is None or num_generations + 1 > cu_seqlens.shape[0]: return None @@ -456,26 +593,38 @@ def apply_device_ragged_layout( verify_lens: torch.Tensor, req_idx: torch.Tensor, kv_correction: torch.Tensor, + kv_lens_delta: Optional[torch.Tensor] = None, ) -> None: """Install device-selected windows and invalidate derived Q prefixes.""" - super().apply_device_ragged_layout(verify_lens, req_idx, kv_correction) + md = self.metadata + super().apply_device_ragged_layout(verify_lens, req_idx, kv_correction, kv_lens_delta) # This call precedes the first model layer. Its fixed-address cumsum is # therefore captured once after the layout update and replayed in order. - self._invalidate_mla_scheduler_buffers() + md.core()._invalidate_mla_scheduler_buffers() + self._invalidate_length_prefixes() + if kv_lens_delta is not None: + num_requests = md.num_seqs + if getattr(self, "cached_token_lens_cuda", None) is not None: + self.cached_token_lens_cuda[:num_requests].copy_( + md.kv_lens_cuda[:num_requests] - md.seq_lens_cuda[:num_requests] + ) + self._refresh_kv_lens_derived_state() + md.clear_lengths_changed() def prepare_for_indexer_k_cache(self): """Prepare the shared indexer K-cache decode table for DSA kernels.""" # INDEXER_COMPRESS uses shared page indices, so the generic DSA # indexer path only needs one 2D block table. - self.kv_cache_manager.copy_batch_indexer_compress_block_tables( + md = self.metadata + md.kv_cache_manager.copy_batch_indexer_compress_block_tables( self.host_indexer_k_cache_block_offsets, - self.request_ids, - beam_width=self.beam_width, - num_contexts=self.num_contexts, - num_seqs=self.num_seqs, + md.request_ids, + beam_width=md.beam_width, + num_contexts=md.num_contexts, + num_seqs=md.num_seqs, ) - self.indexer_k_cache_block_offsets[: self.num_seqs].copy_( - self.host_indexer_k_cache_block_offsets[: self.num_seqs], + self.indexer_k_cache_block_offsets[: md.num_seqs].copy_( + self.host_indexer_k_cache_block_offsets[: md.num_seqs], non_blocking=True, ) # Columns beyond each sequence's allocated indexer blocks contain BAD_PAGE_INDEX (-1). @@ -485,52 +634,54 @@ def prepare_for_indexer_k_cache(self): def prepare_for_block_tables(self): """Prepare block tables for sliding-window and compressed attention.""" - self.kv_cache_manager.copy_batch_sliding_block_tables( + md = self.metadata + md.kv_cache_manager.copy_batch_sliding_block_tables( self.sliding_block_tables, - self.request_ids, - self.num_contexts, - self.num_seqs, + md.request_ids, + md.num_contexts, + md.num_seqs, ) for compress_ratio, compress_block_table in self.compress_block_tables.items(): if ( compress_ratio == DEEPSEEK_V4_SPARSE_RATIO - and self.sparse_metadata_params.enable_kv_cache_offload + and md.sparse_metadata_params.enable_kv_cache_offload ): # Host page indices cannot provide GPU write destinations. # Sparse writes require a separate per-layer arena mapping. compress_block_table.fill_(-1) if self.sparse_offload_state is not None: - self.kv_cache_manager.prepare_sparse_offload( + md.kv_cache_manager.prepare_sparse_offload( self.sparse_offload_state, - self.request_ids, - beam_width=self.beam_width, + md.request_ids, + beam_width=md.beam_width, ) continue - self.kv_cache_manager.copy_batch_compress_block_tables( + md.kv_cache_manager.copy_batch_compress_block_tables( compress_block_table, - self.request_ids, + md.request_ids, compress_ratio=compress_ratio, - beam_width=self.beam_width, - num_contexts=self.num_contexts, - num_seqs=self.num_seqs, + beam_width=md.beam_width, + num_contexts=md.num_contexts, + num_seqs=md.num_seqs, ) for compress_ratio, high_table in (self.mixed_precision_high_block_table or {}).items(): # Any mixed-precision high layer of this ratio shares the pool. self._fill_mixed_precision_high_block_table( - self.kv_cache_manager, + md.kv_cache_manager, self._mixed_precision_high_table_layers[compress_ratio], high_table, ) def prepare_for_deepseek_v4_indices(self, token_positions=None): """Prepare SWA/compressed local indices and sparse_mla_topk_lens.""" + md = self.metadata window_size = self.window_size device = self.swa_local_indices_cuda.device if token_positions is None: # Initial prepare() path — build token_positions from CPU data. - num_tokens = self.num_tokens - num_requests = self.num_seqs + num_tokens = md.num_tokens + num_requests = md.num_seqs # cu_seq_lens_cuda must already be populated before this call cu_seq_lens = self.cu_seq_lens_cuda[: num_requests + 1] @@ -628,7 +779,7 @@ def _prepare_deepseek_v4_indices_compiled( def _build_cache_buffer_data_pointers( self, manager: "DeepseekV4CacheManager", compress_ratios_by_layer: list[int] ) -> tuple[dict[int, int], dict[int, int], dict[int, int]]: - """Build sparse cache pointers for a target or draft manager.""" + """Build sparse cache pointers for ``manager``.""" sparse_mla_base_ptrs = {1: manager.swa_pool_ptr} for ratio, compress_pool_ptr in manager.compress_pool_ptrs.items(): sparse_mla_base_ptrs[ratio] = compress_pool_ptr @@ -650,6 +801,7 @@ def _build_cache_buffer_data_pointers( def _init_cache_buffer_data_pointers(self): # If MTP is enabled, enlarge the compress ratios by max_draft_tokens - 1. + md = self.metadata extend_compress_ratios = self.compress_ratios + [self.compress_ratios[-1]] * ( self.max_draft_tokens - 1 ) @@ -657,69 +809,7 @@ def _init_cache_buffer_data_pointers(self): self.sparse_mla_base_ptrs, self.swa_buffer_ptrs, self.compressed_buffer_ptrs, - ) = self._build_cache_buffer_data_pointers(self.kv_cache_manager, extend_compress_ratios) - - def _init_draft_sparse_buffers(self): - """Initialize sparse buffers for a separate one-model MTP draft cache.""" - self.draft_sliding_block_tables = None - self.draft_sparse_mla_base_ptrs = None - self.draft_swa_buffer_ptrs = None - - draft_mgr = self.draft_kv_cache_manager - from .cache_manager import DeepseekV4CacheManager - - if not isinstance(draft_mgr, DeepseekV4CacheManager): - return - - # Current DSv4 MTP layers are SWA-only. Fail fast if a future model - # introduces a compressed or indexer MTP layer. - draft_ratio = self.compress_ratios[-1] - if draft_ratio != 1: - raise NotImplementedError( - "Separate DeepSeek-V4 draft KV cache supports only SWA-only " - f"(compress_ratio 1) MTP draft layers; got ratio {draft_ratio}." - ) - - draft_block_table_shape = ( - draft_mgr.num_local_layers, - len(DEEPSEEK_V4_SLIDING_ATTENTION), - self.max_num_sequences, - draft_mgr.max_blocks_per_seq, - ) - self.draft_sliding_block_tables = self.get_empty( - self.cuda_graph_buffers, - draft_block_table_shape, - cache_name="draft_sliding_block_tables", - dtype=torch.int32, - capture_graph=self.is_cuda_graph, - ) - extend_compress_ratios = self.compress_ratios + [draft_ratio] * (self.max_draft_tokens - 1) - ( - self.draft_sparse_mla_base_ptrs, - self.draft_swa_buffer_ptrs, - _, - ) = self._build_cache_buffer_data_pointers(draft_mgr, extend_compress_ratios) - - _DRAFT_SPARSE_FIELDS = ( - "sliding_block_tables", - "sparse_mla_base_ptrs", - "swa_buffer_ptrs", - ) - - def prepare_for_draft_forward(self) -> dict | None: - """Repoint sparse fields to the draft buffers for a draft forward.""" - # Draft layers have no mixed-precision high layer while the draft manager is in use. - self.swap_for_draft("mixed_precision_high_block_table", None) - if self.draft_sliding_block_tables is None: - return None - saved_state = {field: getattr(self, field) for field in self._DRAFT_SPARSE_FIELDS} - for field in self._DRAFT_SPARSE_FIELDS: - self.swap_for_draft(field, getattr(self, f"draft_{field}")) - return saved_state - - def restore_after_draft_forward(self, saved_state: dict | None) -> None: - """No-op: restore_draft_swaps() restores the sparse fields, and DSA's hook must not run.""" - return None + ) = self._build_cache_buffer_data_pointers(md.kv_cache_manager, extend_compress_ratios) def validate_sparse_offload_batch(self) -> None: """Reject batches offload cannot run, on the host before any model writes. @@ -732,55 +822,48 @@ def validate_sparse_offload_batch(self) -> None: if self.sparse_offload_state is not None: self._validate_dsa_sparse_offload_batch() + def adopt_target_state(self, source: "DeepseekV4Metadata") -> None: + super().adopt_target_state(source) + # A view of its own buffer, like the target's of the target's. + self.gen_new_tokens_per_seq = ( + None + if source.gen_new_tokens_per_seq is None + else self.gen_new_tokens_per_seq_cuda[: self.metadata.num_generations] + ) + def prepare(self) -> None: + """Derive the DeepSeek-V4 state; replaces the DSA prepare.""" + md = self.metadata + self._invalidate_length_prefixes() + # The indexer schedule is rebuilt below. + self.indexer_schedule_stale = False self._validate_ragged_verification() - assert self.kv_cache_manager is not None - assert self.request_ids is not None + assert md.kv_cache_manager is not None + assert md.request_ids is not None self.validate_sparse_offload_batch() - if self.sparse_offload_state is not None: - self.kv_cache_manager.prepare_sparse_batch(self.request_ids) - - self.kv_cache_manager.compute_sliding_block_tables( - self.request_ids, - self.num_contexts, - ) - - # Prepare the draft manager's tables before the generic prepare copies - # its block offsets, and populate the dedicated DSv4 draft sparse table. - draft_mgr = self.draft_kv_cache_manager - if draft_mgr is not None and hasattr(draft_mgr, "compute_sliding_block_tables"): - draft_mgr.compute_sliding_block_tables( - self.request_ids, - self.num_contexts, - ) - if self.draft_sliding_block_tables is not None: - draft_mgr.copy_batch_sliding_block_tables( - self.draft_sliding_block_tables, - self.request_ids, - self.num_contexts, - self.num_seqs, - ) - - TrtllmAttentionMetadata.prepare(self) - - num_requests = self.num_contexts + self.num_generations - num_gen_tokens = self.num_tokens - self.num_ctx_tokens + num_requests = md.num_contexts + md.num_generations + num_gen_tokens = md.num_tokens - md.num_ctx_tokens self.cached_token_lens_cpu[:num_requests] = torch.tensor( - self.kv_cache_params.num_cached_tokens_per_seq[:num_requests], + md.kv_cache_params.num_cached_tokens_per_seq[:num_requests], dtype=torch.int32, ) cached_token_lens = self.cached_token_lens_cpu - kv_lens = cached_token_lens[:num_requests] + self.seq_lens_kv[:num_requests] + kv_lens = cached_token_lens[:num_requests] + md.seq_lens_kv[:num_requests] kv_lens = self._prepare_device_window_kv_lens(kv_lens) self.cached_token_lens_cuda[:num_requests].copy_( cached_token_lens[:num_requests], non_blocking=True ) + if md.kv_lens_offsets is not None: + # The device cached lengths take the core's KV-length correction, + # so everything derived from them on device below is corrected; + # the host values stay upper bounds. + self.cached_token_lens_cuda[:num_requests] += md.kv_lens_offsets[:num_requests] # Prepare cu_seq_lens early — needed by prepare_for_deepseek_v4_indices - self.cu_seq_lens[1 : num_requests + 1] = self.seq_lens.cumsum(0) + self.cu_seq_lens[1 : num_requests + 1] = md.seq_lens.cumsum(0) self.cu_seq_lens_cuda[: num_requests + 1].copy_( self.cu_seq_lens[: num_requests + 1], non_blocking=True ) @@ -805,14 +888,14 @@ def prepare(self) -> None: # Prepare metadata for indexer (only needed when sparse layers exist) if has_sparse_layers: - DeepseekV4Indexer.prepare(metadata=self) + DeepseekV4Indexer.prepare(metadata=md) # --- Per-ratio metadata --- # 1) CPU-side: compute scalar metadata (num_total_compressed_tokens, etc.) # 2) CUDA-side: fill *_cuda buffers via prepare_compressed_kv_metadata() num_gen_tokens_per_seq = self._sync_gen_tokens_per_seq(num_gen_tokens) - num_contexts = self.num_contexts - num_generations = self.num_generations + num_contexts = md.num_contexts + num_generations = md.num_generations kv_lens_slice = kv_lens[:num_requests] cached_slice = cached_token_lens[:num_requests] @@ -846,13 +929,15 @@ def prepare(self) -> None: ) self.max_ctx_compressed_tokens[compress_ratio] = 0 - # Cached for on_update_kv_lens(); see the reuse gate there. + # Reused by draft steps' refreshes; see _refresh_kv_lens_derived_state. self._ctx_output_sizes = ctx_output_sizes + if md.kv_lens_offsets is not None and md.num_chunked_ctx_requests > 0: + # The correction moved the trailing (extend) context rows' KV + # lengths, so their compressed counts are read on device. + ctx_output_sizes = None # 2) CUDA-side: fill *_cuda buffers on device. - kv_lens_cuda = ( - self.cached_token_lens_cuda[:num_requests] + self._seq_lens_cuda[:num_requests] - ) + kv_lens_cuda = self.cached_token_lens_cuda[:num_requests] + md.seq_lens_cuda[:num_requests] cached_tokens_cuda = self.cached_token_lens_cuda[:num_requests] self.prepare_compressed_kv_metadata(kv_lens_cuda, cached_tokens_cuda, ctx_output_sizes) @@ -864,6 +949,9 @@ def prepare(self) -> None: self.batch_size_cuda, self._compress_ratios_sorted, ) + if md.kv_lens_offsets is not None: + # The DSA indexer state above came from the host KV lengths. + self.on_update_kv_lens() def _sync_gen_tokens_per_seq(self, num_gen_tokens: int) -> int: """Synchronize the compressor's scalar and per-request token counts. @@ -875,16 +963,17 @@ def _sync_gen_tokens_per_seq(self, num_gen_tokens: int) -> int: observes device-window rewrites instead of reloading the captured host shape split. """ - if self.is_ragged_verify and self.num_generations > 0: + md = self.metadata + if self.is_ragged_verify and md.num_generations > 0: self.num_gen_tokens_per_seq = 1 + self.max_draft_tokens - num_contexts = self.num_contexts - num_generations = self.num_generations + num_contexts = md.num_contexts + num_generations = md.num_generations gen_new_tokens = self.gen_new_tokens_per_seq_cuda[:num_generations] - gen_new_tokens.copy_(self._seq_lens_cuda[num_contexts : num_contexts + num_generations]) + gen_new_tokens.copy_(md.seq_lens_cuda[num_contexts : num_contexts + num_generations]) self.gen_new_tokens_per_seq = gen_new_tokens else: self.num_gen_tokens_per_seq = ( - num_gen_tokens // self.num_generations if self.num_generations > 0 else 0 + num_gen_tokens // md.num_generations if md.num_generations > 0 else 0 ) self.gen_new_tokens_per_seq = None return self.num_gen_tokens_per_seq @@ -897,7 +986,7 @@ def prepare_compressed_kv_metadata( ): """Compute per-ratio compressed KV lens and position IDs on device. - Shared by prepare() and on_update_kv_lens() to avoid duplicated logic. + Shared by prepare() and the draft steps' refresh to avoid duplicated logic. Args: kv_lens: Total KV lengths per request (device tensor, [batch_size]). @@ -905,13 +994,15 @@ def prepare_compressed_kv_metadata( ctx_output_sizes: Optional per-ratio host-computed ctx compressed-token counts (Python ints); avoids implicit device-scalar reads (D2H + stream sync) in the ctx position-id - computation. prepare() always passes it; on_update_kv_lens() + computation. prepare() passes it unless the KV-length + correction moved extend-context rows; a draft step's refresh reuses the cached copy unless the extend_ctx path may have mutated ctx-row kv_lens on device. """ + md = self.metadata batch_size = kv_lens.shape[0] - num_contexts = self.num_contexts - num_generations = self.num_generations + num_contexts = md.num_contexts + num_generations = md.num_generations self._compute_per_ratio_kv_lens( kv_lens, @@ -947,24 +1038,28 @@ def prepare_compressed_kv_metadata( self._compress_ratios_sorted, ) - def on_update_kv_lens(self, skip_indexer_schedule: bool = False): + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + self._refresh_kv_lens_derived_state() + + def _refresh_kv_lens_derived_state(self) -> None: """Recompute kv-lens-dependent DeepSeek-V4 metadata on device.""" - super().on_update_kv_lens(skip_indexer_schedule=skip_indexer_schedule) + md = self.metadata - batch_size = self.num_seqs - num_tokens = self.num_tokens - kv_lens = self.kv_lens_cuda[:batch_size] - seq_lens = self._seq_lens_cuda[:batch_size] + batch_size = md.num_seqs + num_tokens = md.num_tokens + kv_lens = md.kv_lens_cuda[:batch_size] + seq_lens = md.seq_lens_cuda[:batch_size] cached_tokens = kv_lens - seq_lens - num_gen_tokens = num_tokens - self.num_ctx_tokens + num_gen_tokens = num_tokens - md.num_ctx_tokens self._sync_gen_tokens_per_seq(num_gen_tokens) # Reuse prepare()'s host-computed ctx sizes unless the extend_ctx path # (num_chunked_ctx_requests > 0) may have mutated ctx-row kv_lens on # device; every other path only changes gen rows. ctx_output_sizes = ( - self._ctx_output_sizes if getattr(self, "num_chunked_ctx_requests", 0) == 0 else None + self._ctx_output_sizes if getattr(md, "num_chunked_ctx_requests", 0) == 0 else None ) self.prepare_compressed_kv_metadata(kv_lens, cached_tokens, ctx_output_sizes) @@ -983,7 +1078,7 @@ def on_update_kv_lens(self, skip_indexer_schedule: bool = False): batch_size, num_tokens, self.cu_seq_lens_cuda, - self.req_idx_per_token, + self.index_share.req_idx_per_token, ) self.prepare_for_deepseek_v4_indices(token_positions) @@ -1173,3 +1268,9 @@ def _compute_ctx_compressed_position_ids( compressed_position_ids_bufs[compress_ratio][:total_ctx_comp] = ( (past_kv[:num_contexts][ctx_req] + ctx_offset) * compress_ratio ).to(torch.int) + + +class DeepseekV4TrtllmAttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the DeepSeek-V4 extension attached.""" + + backend_extension_types = (DeepseekV4Metadata,) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/__init__.py index b0a87e1ce3c6..c8db245aa94c 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/__init__.py @@ -24,7 +24,7 @@ split_prefill_chunks, transform_local_topk_and_prepare_pool_view, ) -from .metadata import DSAtrtllmAttentionMetadata, build_req_idx_per_token +from .metadata import DSAMetadata, DSAtrtllmAttentionMetadata, build_req_idx_per_token from .params import DSABackendForwardArgs, DSAMetadataParams, DSAParams from .vanilla_backend import DSAVanillaAttention, DSAVanillaIndexer @@ -33,6 +33,7 @@ "DSABackendForwardArgs", "DSACacheManager", "DSACacheManagerV2", + "DSAMetadata", "DSAMetadataParams", "DSAParams", "DSATrtllmAttention", diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py index d07efa36fde8..6580bb9bf79d 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py @@ -82,7 +82,18 @@ if TYPE_CHECKING: from tensorrt_llm.mapping import Mapping - from .metadata import DSAtrtllmAttentionMetadata + from .metadata import DSAMetadata, DSAtrtllmAttentionMetadata + + +def _dsa_state(metadata: "DSAtrtllmAttentionMetadata") -> "DSAMetadata": + """The DSA extension of ``metadata``. + + Imported on use: the metadata module imports this one. + """ + from .metadata import DSAMetadata + + return metadata.ext(DSAMetadata) + # Optional import: fast-hadamard-transform causes CI build issues (requires wheel+torch pre-installed) try: @@ -309,31 +320,32 @@ def transform_local_topk_and_prepare_pool_view( Uses cached values from attn_metadata._ensure_pool_view_cached() to avoid redundant Python/CUDA overhead across layers. """ + dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - attn_metadata._ensure_pool_view_cached() + dsa._ensure_pool_view_cached() page_index_scale, layer_offset = ( attn_metadata.kv_cache_manager.get_primary_pool_page_index_params(layer_idx) ) if is_generation: - block_table = attn_metadata._cached_block_table_gen - req_idx = attn_metadata._cached_req_idx_gen + block_table = dsa._cached_block_table_gen + req_idx = dsa._cached_req_idx_gen else: - block_table = attn_metadata._cached_block_table_ctx - req_idx = attn_metadata._cached_req_idx_ctx + block_table = dsa._cached_block_table_ctx + req_idx = dsa._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global( req_idx, block_table, topk_indices, - attn_metadata._cached_tokens_per_block, + dsa._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * attn_metadata._cached_tokens_per_block, + page_index_scale * dsa._cached_tokens_per_block, layer_offset, ) - return global_indices, attn_metadata._cached_pool_view + return global_indices, dsa._cached_pool_view def transform_local_topk_and_prepare_pool_view_grouped( @@ -358,24 +370,25 @@ def transform_local_topk_and_prepare_pool_view_grouped( is bit-identical to ``transform_local_topk_and_prepare_pool_view`` for the member with offset ``layer_ids[z]``. MLA-only path. """ + dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - attn_metadata._ensure_pool_view_cached() + dsa._ensure_pool_view_cached() if is_generation: - block_table = attn_metadata._cached_block_table_gen - req_idx = attn_metadata._cached_req_idx_gen + block_table = dsa._cached_block_table_gen + req_idx = dsa._cached_req_idx_gen else: - block_table = attn_metadata._cached_block_table_ctx - req_idx = attn_metadata._cached_req_idx_ctx + block_table = dsa._cached_block_table_ctx + req_idx = dsa._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global_grouped( req_idx, block_table, topk_indices, - attn_metadata._cached_tokens_per_block, + dsa._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * attn_metadata._cached_tokens_per_block, + page_index_scale * dsa._cached_tokens_per_block, layer_ids, ) @@ -1170,6 +1183,7 @@ def prepare_one_prefill_chunk( @staticmethod def build_indexer_params(metadata: DSAtrtllmAttentionMetadata) -> Optional[IndexerParams]: + dsa = _dsa_state(metadata) kv_cache_manager = metadata.kv_cache_manager if kv_cache_manager is None or not hasattr(kv_cache_manager, "index_head_dim"): return None @@ -1187,7 +1201,7 @@ def build_indexer_params(metadata: DSAtrtllmAttentionMetadata) -> Optional[Index num_ctx_tokens=metadata.num_ctx_tokens, head_dim=head_dim, quant_block_size=metadata.indexer_quant_block_size, - tokens_per_block=metadata._tokens_per_block, + tokens_per_block=dsa._tokens_per_block, compress_ratio=compress_ratio, request_ids=metadata.request_ids, num_past_tokens=metadata.kv_cache_params.num_cached_tokens_per_seq, @@ -1430,14 +1444,12 @@ def prepare_scheduler_metadata(metadata: DSAtrtllmAttentionMetadata): """ Prepare scheduler metadata for the DeepGEMM decode MQA kernel. """ + dsa = _dsa_state(metadata) num_contexts = metadata.num_contexts num_generations = metadata.num_generations - # A full rebuild clears the stale marker. - metadata.indexer_schedule_stale = False - gen_seq_lens = metadata.get_indexer_kv_lens( + gen_seq_lens = dsa.bind_gen_indexer_kv_lens( metadata.kv_lens_cuda_runtime[num_contexts : num_contexts + num_generations] ) - metadata.gen_indexer_kv_lens_cuda_runtime = gen_seq_lens if not metadata.use_expanded_buffers_for_mtp: scheduler_context_lens = metadata.update_indexer_kv_lens_2d(gen_seq_lens) scheduler_metadata_buffer = get_paged_mqa_logits_metadata( diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py index deef67d9c3af..fdde42abb484 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py @@ -6,13 +6,14 @@ import functools import os -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar, FrozenSet, List, Optional, Tuple +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Tuple import torch import tensorrt_llm import tensorrt_llm.bindings +from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from tensorrt_llm._torch.utils import maybe_compile @@ -20,12 +21,13 @@ from tensorrt_llm.deep_gemm import get_paged_mqa_logits_metadata from tensorrt_llm.logger import logger +from ...features.spec_decoding import SpecDecodingMetadata from ..offload import ( SparseOffloadState, allocate_sparse_offload_state, get_sparse_offload_metadata_layers, ) -from ..params import use_self_sampling_gvr +from ..params import MTPIndexShareUser, SparseAlgorithmExtension, use_self_sampling_gvr from .cache_manager import is_dsa_cache_manager from .indexer import ( _DG_SCHEDULE_BLOCK_KV, @@ -50,8 +52,6 @@ if TYPE_CHECKING: from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 - from tensorrt_llm._torch.speculative.interface import SpecMetadata - from tensorrt_llm._torch.speculative.spec_tree_manager import SpecTreeManager @functools.lru_cache(maxsize=1) @@ -96,196 +96,392 @@ class TokenMajorGenView: max_num_rows: int -@dataclass(init=False) -class DSAtrtllmAttentionMetadata(TrtllmAttentionMetadata): - """Attention metadata for DSA (Dense Sparse Attention) with indexer state.""" - - # Mixed-precision high page tables; -1 = not in the high pool. - mixed_precision_high_block_table = None - draft_mixed_precision_high_block_table = None - sparse_metadata_params: Optional[DSAMetadataParams] = None - use_fp8_ds_mla: bool = field(default=False, init=False) - # Store reference to indexer for preparation stage - indexer: Optional["Indexer"] = None - # Chunked prefill metadata for indexer (prefill-only, no CUDA graph needed) - indexer_prefill_chunks: Optional[List[IndexerPrefillChunkMetadata]] = None - # (local q_begin, local q_end, per-rank sizes) when indexer_prefill_chunks - # cover only this TP rank's share of the context query tokens; the top-k - # results are then all-gathered once after the chunks. - indexer_prefill_q_split: Optional[Tuple[int, int, List[int]]] = None - # Max chunk size for two-level chunking: - # 1. Request-level: Pack multiple small requests into one chunk (up to indexer_max_chunk_size) - # 2. Intra-request: Split large requests into Q-blocks when seq_len > max_chunk_size - indexer_max_chunk_size: int - # TopK for static token sparse attention - num_sparse_topk: int - # TopK for dynamic sparse MLA - sparse_mla_topk: int - radix_aux_indices: Optional[torch.Tensor] = field(default=None, init=False) - radix_aux_logits: Optional[torch.Tensor] = field(default=None, init=False) - # max number of draft tokens - max_draft_tokens: int = 0 - # Indexer head dimension - indexer_head_dim: int = 128 - # Elements per four-byte scale word, matching Indexer.prepare(). - indexer_quant_block_size: int = 128 - # Enable indexer skip for short sequences - enable_indexer_skip: bool = False - shared_topk_indices: Optional[torch.Tensor] = None - indexer_skip_topk: bool = False - in_mtp_draft_loop: bool = False - mtp_num_accepted: Optional[torch.Tensor] = None - # Set by on_update_kv_lens(skip_indexer_schedule=True): the DeepGEMM MQA-logits schedule, the - # 2D/expanded indexer KV lengths and the GVR row order were left untouched. Legal only if no - # MQA-logits + TopK pass runs before the next full rebuild; the indexer asserts via this marker. - indexer_schedule_stale: bool = False - # Whether skip the indexer for context requests - skip_indexer_for_ctx_reqs: bool = False - # Whether skip the indexer for generation requests - skip_indexer_for_gen_reqs: bool = False - # Whether to use the expanded buffers for MTP support - use_expanded_buffers_for_mtp: bool = False - # Whether to reshape the DSL paged MQA logits Q tensor into a kernel- - # supported `effective_next_n` via caller-side atom-split (FP4: {1,2,3}; - # FP8: {1,2,3,4}; see `_pick_dsl_expand`). Reuses - # `kv_lens_expanded_cuda` / `block_table_expanded` / - # `scheduler_metadata_buffer_expanded`; runtime mutually exclusive with - # `use_expanded_buffers_for_mtp` (the latter requires `not _use_dsl`). - expand_for_dsl: bool = False - # Cached (expand_factor, atom) decision from the wave-aware picker. Set at - # `prepare()` time and read by forward call sites — avoids re-running the - # picker per call and guarantees prepare/forward use the SAME decision - # (otherwise the populated buffers would mismatch the kernel reshape). - dsl_expand_factor: int = 1 - dsl_atom: int = 1 - # In-kernel uneven split (``_pick_dsl_split``): atom count, the window it - # was picked for, and the per-atom causal stagger on device. Its schedule - # reuses ``kv_lens_expanded_cuda`` / ``scheduler_metadata_buffer_expanded``. - dsl_kernel_atoms: int = 1 - dsl_kernel_next_n: int = 0 - dsl_atom_ctx_offsets: Optional[torch.Tensor] = None - dsl_atom_ctx_offsets_key: Tuple[int, int] = (0, 0) - # Compression ratio for KV tokens - compress_ratios: List[int] = field(default_factory=lambda: [1]) - # Number of compressed KV tokens for context requests - num_ctx_kv_tokens: int = 0 - gen_indexer_kv_lens_cuda_runtime: Optional[torch.Tensor] = None - # Rebound by on_update_kv_lens(); the decode graph recomputes it. - graph_temporary_attrs: ClassVar[FrozenSet[str]] = frozenset( - {"gen_indexer_kv_lens_cuda_runtime"} +class DSAMetadata(SparseAlgorithmExtension, MTPIndexShareUser): + """Indexer state of DSA (Dense Sparse Attention) on a TRTLLM metadata.""" + + fields = frozenset( + { + "_cached_block_table_ctx", + "_cached_block_table_gen", + "_cached_num_pool_tokens", + "_cached_pool_view", + "_cached_req_idx_ctx", + "_cached_req_idx_gen", + "_cached_tokens_per_block", + "_fused_dsa_meta_armed", + "_group_remap_batched", + "_indexer_compress_ratio", + "_ragged_mla_buffers_valid", + "_tokens_per_block", + "attn_row_block_offsets", + "attn_row_kv_correction_cuda", + "attn_row_kv_correction_host", + "attn_row_kv_lens_cuda", + "attn_row_kv_lens_host", + "attn_row_prompt_lens_cpu", + "attn_row_prompt_lens_cuda", + "attn_row_req_idx_cuda", + "attn_row_req_idx_host", + "attn_row_request_types_host", + "block_table_expanded", + "compress_ratios", + "cu_seqlen_ke", + "cu_seqlen_ks", + "device_windows_mode", + "dsl_atom", + "dsl_atom_ctx_offsets", + "dsl_atom_ctx_offsets_key", + "dsl_expand_factor", + "dsl_kernel_atoms", + "dsl_kernel_next_n", + "enable_gvr_topk", + "enable_indexer_skip", + "enable_ragged_verification", + "expand_for_dsl", + "gen_cached_token_indptr", + "gen_indexer_kv_lens_cuda_runtime", + "gen_kv_indptr", + "gen_token_repeats_cuda", + "gen_token_stride", + "gvr_prior_indices", + "has_mixed_precision_high_rows", + "host_block_table_expanded", + "host_gen_cached_token_indptr", + "host_gen_kv_indptr", + "host_gen_token_repeats", + "host_indexer_k_cache_block_offsets", + "host_req_idx_per_token", + "host_slot_mapping_fp8", + "host_slot_mapping_scale", + "host_topk_indices_buffer", + "indexer", + "indexer_head_dim", + "indexer_k_cache_block_offsets", + "indexer_max_chunk_size", + "indexer_prefill_chunks", + "indexer_prefill_q_split", + "indexer_quant_block_size", + "indexer_schedule_stale", + "_inline_scale_pool", + "is_ragged_verify", + "kv_lens_cuda_2d", + "kv_lens_expanded_cuda", + "kv_lens_expanded_host", + "kv_lens_row_reorder", + "kv_lens_row_reorder_buffer", + "max_draft_tokens", + "max_gen_seq_len", + "mixed_precision_high_block_table", + "mixed_precision_staging_enabled", + "needs_gvr_prior", + "num_ctx_kv_tokens", + "num_ctx_mla_kv_tokens", + "num_sms", + "nvfp4_mla_compact_indices", + "nvfp4_mla_context_fp8_scratch", + "nvfp4_mla_fp8_scratch", + "radix_aux_indices", + "radix_aux_logits", + "ragged_mla_cu_kv_seqlens", + "ragged_mla_cu_q_rows", + "ragged_verify_lens", + "row_kv_correction_cuda", + "row_kv_correction_host", + "row_kv_lens_cuda", + "row_kv_lens_host", + "row_req_idx_cuda", + "row_req_idx_host", + "runtime_tokens_per_gen_step", + "scheduler_metadata_buffer", + "scheduler_metadata_buffer_expanded", + "scheduler_metadata_buffer_full_next_n", + "skip_indexer_for_ctx_reqs", + "skip_indexer_for_gen_reqs", + "slot_mapping_fp8", + "slot_mapping_fp8_fullkv", + "slot_mapping_scale", + "slot_mapping_scale_fullkv", + "sparse_latent_block_offsets", + "sparse_latent_write_slots", + "sparse_mla_topk", + "sparse_offload_attn_indices", + "sparse_offload_fetch_event", + "sparse_offload_fetch_pending", + "sparse_offload_next_shared_layer", + "sparse_offload_start_event", + "sparse_offload_state", + "sparse_offload_stream", + "sparse_offload_write_event", + "sparse_offload_write_pending", + "token_positions_cuda", + "topk_indices_buffer", + "use_cute_dsl_paged_mqa_logits", + "use_cute_dsl_topk", + "use_expanded_buffers_for_mtp", + "use_fp8_ds_mla", + "use_self_sampling_topk", + } + ) + methods = frozenset( + { + "_compute_kv_lens_row_reorder", + "_create_kv_lens_2d_buffer", + "_ensure_group_remap_struct", + "_ensure_pool_view_cached", + "_invalidate_pool_view_cache", + "mla_scheduler_buffers", + "token_major_gen_view", + "apply_device_ragged_layout", + "create_buffers_for_indexer", + "create_buffers_for_mla_rope_append", + "create_expanded_buffers", + "expand_per_gen_token", + "gen_token_repeat_list", + "gen_token_repeats", + "get_indexer_kv_lens", + "get_indexer_max_seq_len", + "prepare_dense_topk_indices", + "prepare_for_indexer_k_cache", + "prepare_for_indices_conversion", + "prepare_for_mla_rope_append", + "prepare_for_skip_indexer", + "prepare_for_spec_decode", + "ragged_row_kv_lens", + "refresh_ragged_row_kv_lens", + "refresh_token_major_block_table", + "refresh_token_major_gen_rows", + "set_gen_token_layout", + "set_ragged_verify_lens", + "update_indexer_kv_lens_2d", + "warmup_cute_dsl_radix_topk", + "warmup_selfsampling_topk", + } + ) + # Whether the ragged-verification buffers are allocated (DSpark + # confidence scheduling); fixed before CUDA graph metadata is created. + init_fields = { + # Allocate the row-major and token-major buffers used only by ragged + # verification. This is fixed before CUDA-graph metadata is created. + "enable_ragged_verification": False, + } + + adopts_target_state = True + # A draft binds the target's indexer state: settings, host staging, block + # tables, prefill chunks and the step's derived lengths. + draft_bound = frozenset( + { + "_attn_num_rows", + "_draft_alloc_cap", + "_expanded_alloc_draft_cap", + "_group_remap_struct", + "_group_remap_struct_kv_id", + "_indexer_compress_ratio", + "_radix_rows_per_sequence", + "_ragged_num_rows", + "_tokens_per_block", + "attn_row_block_offsets", + "attn_row_kv_correction_cuda", + "attn_row_kv_correction_host", + "attn_row_kv_lens_host", + "attn_row_prompt_lens_cpu", + "attn_row_prompt_lens_cuda", + "attn_row_req_idx_cuda", + "attn_row_req_idx_host", + "attn_row_request_types_host", + "compress_ratios", + "cu_seqlen_ke", + "cu_seqlen_ks", + "device_windows_mode", + "dsl_atom", + "dsl_atom_ctx_offsets", + "dsl_atom_ctx_offsets_key", + "dsl_expand_factor", + "dsl_kernel_atoms", + "dsl_kernel_next_n", + "enable_gvr_topk", + "enable_indexer_skip", + "enable_ragged_verification", + "expand_for_dsl", + "gen_token_repeats_cuda", + "host_block_table_expanded", + "host_gen_cached_token_indptr", + "host_gen_kv_indptr", + "host_gen_token_repeats", + "host_indexer_k_cache_block_offsets", + "host_req_idx_per_token", + "host_slot_mapping_fp8", + "host_slot_mapping_scale", + "host_topk_indices_buffer", + "indexer", + "indexer_head_dim", + "indexer_max_chunk_size", + "indexer_prefill_chunks", + "indexer_prefill_q_split", + "indexer_quant_block_size", + "kv_lens_expanded_host", + "max_draft_tokens", + "max_gen_seq_len", + "mixed_precision_high_block_table", + "needs_gvr_prior", + "num_ctx_kv_tokens", + "num_ctx_mla_kv_tokens", + "num_sms", + "ragged_verify_lens", + "row_kv_correction_cuda", + "row_kv_correction_host", + "row_kv_lens_host", + "row_req_idx_cuda", + "row_req_idx_host", + "runtime_tokens_per_gen_step", + "skip_indexer_for_ctx_reqs", + "skip_indexer_for_gen_reqs", + "sparse_metadata_params", + "sparse_mla_topk", + "use_cute_dsl_topk", + "use_expanded_buffers_for_mtp", + "use_fp8_ds_mla", + "use_self_sampling_topk", + } + ) + # Rewritten in place by the length refresh a draft loop runs, and the + # indexer block offsets the pool-index transform clamps in place. + draft_written = frozenset( + { + "attn_row_kv_lens_cuda", + "block_table_expanded", + "gen_cached_token_indptr", + "gen_kv_indptr", + "indexer_k_cache_block_offsets", + "kv_lens_cuda_2d", + "kv_lens_expanded_cuda", + "kv_lens_row_reorder_buffer", + "row_kv_lens_cuda", + "scheduler_metadata_buffer", + "scheduler_metadata_buffer_expanded", + "scheduler_metadata_buffer_full_next_n", + "slot_mapping_fp8", + "slot_mapping_scale", + "token_positions_cuda", + "topk_indices_buffer", + } + ) + # Scratch and selection state the forward builds or carries across steps, + # and views a draft points at its own buffers (adopt_target_state). + draft_private = frozenset( + { + "nvfp4_mla_context_fp8_scratch", + "indexer_schedule_stale", + "_ragged_mla_buffers_valid", + "_pool_cache_valid", + "_group_remap_batched", + "_cached_tokens_per_block", + "_cached_req_idx_gen", + "_cached_req_idx_ctx", + "_cached_pool_view", + "_cached_num_pool_tokens", + "_cached_kv_mgr_id", + "_cached_block_table_gen", + "_cached_block_table_ctx", + "_fused_dsa_meta_armed", + "gen_indexer_kv_lens_cuda_runtime", + "_gen_indexer_kv_lens_buffer", + "gvr_prior_indices", + "kv_lens_row_reorder", + "nvfp4_mla_compact_indices", + "nvfp4_mla_fp8_scratch", + "radix_aux_indices", + "radix_aux_logits", + "ragged_mla_cu_kv_seqlens", + "ragged_mla_cu_q_rows", + "slot_mapping_fp8_fullkv", + "slot_mapping_scale_fullkv", + } ) - # Temporal-GVR prior state: allocated only when the two-level dispatch - # selects the temporal engine (the self-sampling engine keeps no - # cross-step state). - needs_gvr_prior: bool = field(default=False, init=False) - gvr_prior_indices: Optional[torch.Tensor] = field(default=None, init=False) - # Query tokens per generation request this step, in batch order. None - # denotes the uniform runtime stride. - ragged_verify_lens: Optional[List[int]] = None - # Allocate the row-major and token-major buffers used only by ragged - # verification. This is fixed before CUDA-graph metadata is created. - enable_ragged_verification: bool = False - # In device-window mode the host split determines only the captured shape; - # the true windows are installed before replay. - device_windows_mode: bool = False - # Query tokens per generation request for a uniform runtime tier. Zero - # falls back to the static draft-token ceiling. - runtime_tokens_per_gen_step: int = 0 - - @property - def is_ragged_verify(self) -> bool: - return self.ragged_verify_lens is not None - - def _validate_ragged_verification(self) -> None: - """Reject layouts whose stable buffers were not enabled at allocation.""" - if self.ragged_verify_lens is not None and not self.enable_ragged_verification: - raise RuntimeError( - "ragged verification metadata was installed without enabling " - "ragged-verification buffers" - ) - - @property - def gen_token_stride(self) -> int: - stride = int(self.runtime_tokens_per_gen_step) - return stride if stride > 0 else 1 + self.max_draft_tokens - - def update_indexer_kv_lens_2d(self, gen_kv_lens: torch.Tensor) -> torch.Tensor: - """Fill and return the active query-width view of persistent storage.""" - width = min(self.gen_token_stride, self.kv_lens_cuda_2d.shape[1]) - context_lens = self.kv_lens_cuda_2d[: self.num_generations, :width] - context_lens.copy_(gen_kv_lens.unsqueeze(-1).expand(-1, width)) - return context_lens - - def mla_prepare_scheduler_buffers(self, num_heads: int) -> tuple[torch.Tensor, torch.Tensor]: - """Precompute prefixes for the same rows presented to MLA generation.""" - view = self.token_major_gen_view() - if view is None: - return super().mla_prepare_scheduler_buffers(num_heads) - - num_ctx = self.num_contexts - n_gen = view.num_rows - num_ctx - if not self._mla_scheduler_buffers_valid: - # RoPE presents one query token per generation row. The device - # extents are causal, pre-extra-KV lengths, just like the - # cache_seq_lens scanned by the non-precomputed kernel path. - torch.cumsum( - view.prompt_lens_cuda[num_ctx:], - 0, - dtype=torch.int32, - out=self.ragged_mla_cu_q_rows[1 : n_gen + 1], - ) - self.ragged_mla_cu_q_rows[1 : n_gen + 1] *= num_heads - torch.cumsum( - view.sequence_length[num_ctx:], - 0, - dtype=torch.int32, - out=self.ragged_mla_cu_kv_seqlens[1 : n_gen + 1], - ) - self._mla_scheduler_buffers_valid = True - return ( - self.ragged_mla_cu_q_rows[: n_gen + 1], - self.ragged_mla_cu_kv_seqlens[: n_gen + 1], - ) - - def gen_token_repeat_list(self) -> List[int]: - if not self.is_ragged_verify: - return [self.gen_token_stride] * self.num_generations - return list(self.ragged_verify_lens) - - def gen_token_repeats(self, device: Optional[torch.device] = None) -> torch.Tensor: - return torch.tensor(self.gen_token_repeat_list(), dtype=torch.long, device=device) - - def expand_per_gen_token(self, values: torch.Tensor, dim: int = 0) -> Tuple[torch.Tensor, int]: - """Repeat request values once per generation query token.""" - repeats = self.gen_token_repeat_list() - num_tokens = sum(repeats) - if not self.is_ragged_verify: - return values.repeat_interleave(self.gen_token_stride, dim=dim), num_tokens - if values.device == self.gen_token_repeats_cuda.device: - repeats_dev = self.gen_token_repeats_cuda[: self.num_generations] - else: - repeats_dev = torch.tensor(repeats, dtype=torch.long, device=values.device) - return ( - values.repeat_interleave(repeats_dev, dim=dim, output_size=num_tokens), - num_tokens, - ) - def __init__(self, *args, **kwargs): - """Initialize DSA metadata with SM count and indexer chunk size.""" - self.enable_ragged_verification = bool(kwargs.pop("enable_ragged_verification", False)) - sparse_attention_config = kwargs.pop("sparse_attention_config", None) - if ( - kwargs.get("sparse_metadata_params") is None - and sparse_attention_config is not None - and hasattr(sparse_attention_config, "to_sparse_metadata_params") - ): - kwargs["sparse_metadata_params"] = sparse_attention_config.to_sparse_metadata_params() + # (id(kv_cache_manager), paged view) of the inline-scale primary pool the + # FlashInfer sparse-MLA path reads; rebuilt when the manager changes. + _inline_scale_pool: Optional[Tuple[int, torch.Tensor]] = None + # Read by the core through ``md.find``: a mixed-precision KV cache gets no + # staging copy; attention reads the sparse gather's FP8 scratch and + # overlays the high pool's rows on it. + mixed_precision_staging_enabled = False + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.use_fp8_ds_mla = False + # Store reference to indexer for preparation stage + self.indexer: Optional["Indexer"] = None + # Chunked prefill metadata for indexer (prefill-only, no CUDA graph needed) + self.indexer_prefill_chunks: Optional[List[IndexerPrefillChunkMetadata]] = None + # (local q_begin, local q_end, per-rank sizes) when indexer_prefill_chunks + # cover only this TP rank's share of the context query tokens; the top-k + # results are then all-gathered once after the chunks. + self.indexer_prefill_q_split: Optional[Tuple[int, int, List[int]]] = None + # Max chunk size for two-level chunking: + # 1. Request-level: Pack multiple small requests into one chunk (up to indexer_max_chunk_size) + # 2. Intra-request: Split large requests into Q-blocks when seq_len > max_chunk_size + self.indexer_max_chunk_size = 0 + # TopK for dynamic sparse MLA + self.sparse_mla_topk = 0 + self.radix_aux_indices: Optional[torch.Tensor] = None + self.radix_aux_logits: Optional[torch.Tensor] = None + # max number of draft tokens + self.max_draft_tokens = 0 + # Indexer head dimension + self.indexer_head_dim = 128 + # Elements per four-byte scale word, matching Indexer.prepare(). + self.indexer_quant_block_size = 128 + # Enable indexer skip for short sequences + self.enable_indexer_skip = False + # Whether skip the indexer for context requests + self.skip_indexer_for_ctx_reqs = False + # Whether skip the indexer for generation requests + self.skip_indexer_for_gen_reqs = False + # Whether to use the expanded buffers for MTP support + self.use_expanded_buffers_for_mtp = False + # Whether to reshape the DSL paged MQA logits Q tensor into a kernel- + # supported `effective_next_n` via caller-side atom-split (FP4: {1,2,3}; + # FP8: {1,2,3,4}; see `_pick_dsl_expand`). Reuses + # `kv_lens_expanded_cuda` / `block_table_expanded` / + # `scheduler_metadata_buffer_expanded`; runtime mutually exclusive with + # `use_expanded_buffers_for_mtp` (the latter requires `not _use_dsl`). + self.expand_for_dsl = False + # Cached (expand_factor, atom) decision from the wave-aware picker. Set at + # `prepare()` time and read by forward call sites — avoids re-running the + # picker per call and guarantees prepare/forward use the SAME decision + # (otherwise the populated buffers would mismatch the kernel reshape). + self.dsl_expand_factor = 1 + self.dsl_atom = 1 + # In-kernel uneven split (``_pick_dsl_split``): atom count, the window it + # was picked for, and the per-atom causal stagger on device. Its schedule + # reuses ``kv_lens_expanded_cuda`` / ``scheduler_metadata_buffer_expanded``. + self.dsl_kernel_atoms = 1 + self.dsl_kernel_next_n = 0 + self.dsl_atom_ctx_offsets: Optional[torch.Tensor] = None + self.dsl_atom_ctx_offsets_key: Tuple[int, int] = (0, 0) + # Compression ratio for KV tokens + self.compress_ratios: List[int] = [1] + # Number of compressed KV tokens for context requests + self.num_ctx_kv_tokens = 0 + self.gen_indexer_kv_lens_cuda_runtime: Optional[torch.Tensor] = None + # Temporal-GVR prior state: allocated only when the two-level dispatch + # selects the temporal engine (the self-sampling engine keeps no + # cross-step state). + self.needs_gvr_prior = False + self.gvr_prior_indices: Optional[torch.Tensor] = None + # Query tokens per generation request this step, in batch order. None + # denotes the uniform runtime stride. + self.ragged_verify_lens: Optional[List[int]] = None + # In device-window mode the host split determines only the captured shape; + # the true windows are installed before replay. + self.device_windows_mode = False + # Query tokens per generation request for a uniform runtime tier. Zero + # falls back to the static draft-token ceiling. + self.runtime_tokens_per_gen_step = 0 self.num_sms = tensorrt_llm.deep_gemm.get_num_sms() # Cached step-invariant values for transform_local_topk_and_prepare_pool_view. # These are recomputed once per step in _ensure_pool_view_cached() and # reused across all layers to avoid redundant Python/CUDA overhead. - # Initialized here as plain instance attributes (not class-level - # annotations) to stay invisible to dataclass/torch.compile introspection. self._pool_cache_valid = False self._cached_kv_mgr_id = 0 self._cached_pool_view = None @@ -303,47 +499,47 @@ def __init__(self, *args, **kwargs): self._group_remap_struct_kv_id = 0 self._group_remap_batched = {} self.num_ctx_mla_kv_tokens = 0 + self.max_gen_seq_len = 0 self.nvfp4_mla_context_fp8_scratch = None self._radix_rows_per_sequence = 1 - super().__init__(*args, **kwargs) - sparse_metadata_params = self.sparse_metadata_params - if not isinstance(sparse_metadata_params, DSAMetadataParams): - raise ValueError("DSA sparse attention metadata params are not set") - self.indexer_max_chunk_size = sparse_metadata_params.indexer_max_chunk_size - - @property - def use_cute_dsl_paged_mqa_logits(self) -> bool: - """The Indexer's decision: the config flag gated on SM100 and cutlass-dsl.""" - return use_cute_dsl_paged_mqa_logits( - self.sparse_metadata_params.use_cute_dsl_paged_mqa_logits - ) - - def __post_init__(self): - """Allocate indexer K-cache buffers and heuristic TopK metadata.""" - super().__post_init__() - if self.is_cross: + self._ragged_mla_buffers_valid = False + # Set by a draft refresh that skipped the indexer schedule + # rebuild (an index-sharing draft step reuses its TopK selection): the + # DeepGEMM MQA-logits schedule, the 2D/expanded indexer KV lengths and + # the GVR row order are stale until the next prepare. The indexer + # raises if a TopK pass runs on them. + self.indexer_schedule_stale = False + # Mixed-precision high page table of this metadata's KV cache manager, + # [max_num_sequences, max_blocks_per_seq], -1 = not in the high pool; + # None without a mixed-precision high pool. + self.mixed_precision_high_block_table: Optional[torch.Tensor] = None + md = self.metadata + if md.is_cross: # Ragged verification is a target self-attention capability. A # cross-attention metadata object may be a shallow copy of one, but # must not inherit its verifier-only buffers or per-step layout. self.enable_ragged_verification = False self.ragged_verify_lens = None self._validate_ragged_verification() - if not is_dsa_cache_manager(self.kv_cache_manager): + if md.kv_cache_manager is None and self.sparse_metadata_params is None: + return + if not is_dsa_cache_manager(md.kv_cache_manager): has_deepseek_v4_cache_interface = all( - hasattr(self.kv_cache_manager, attr) + hasattr(md.kv_cache_manager, attr) for attr in ("compressed_block_sizes", "get_cache_indices") ) assert has_deepseek_v4_cache_interface, ( - "DSAtrtllmAttentionMetadata requires DSACacheManager-compatible " - f"cache manager, got {type(self.kv_cache_manager)}" + "DSAMetadata requires DSACacheManager-compatible " + f"cache manager, got {type(md.kv_cache_manager)}" ) - self.use_fp8_ds_mla = getattr(self.kv_cache_manager, "use_fp8_ds_mla", False) + self.use_fp8_ds_mla = getattr(md.kv_cache_manager, "use_fp8_ds_mla", False) sparse_metadata_params = self.sparse_metadata_params if not isinstance(sparse_metadata_params, DSAMetadataParams): raise ValueError("DSA sparse attention metadata params are not set") - self.num_sparse_topk = sparse_metadata_params.max_sparse_topk - self.sparse_mla_topk = self.num_sparse_topk + self.indexer_max_chunk_size = sparse_metadata_params.indexer_max_chunk_size + md.num_sparse_topk = sparse_metadata_params.max_sparse_topk + self.sparse_mla_topk = md.num_sparse_topk self.indexer_head_dim = sparse_metadata_params.index_head_dim self.indexer_quant_block_size = 128 self.enable_indexer_skip = sparse_metadata_params.enable_indexer_skip @@ -354,7 +550,7 @@ def __post_init__(self): sparse_metadata_params.enable_heuristic_topk and get_sm_version() >= 100 ) self.kv_lens_row_reorder = None - capture_graph = self.is_cuda_graph + capture_graph = md.is_cuda_graph # Plain DSA has no compression and uses the default [1]. DeepSeek-V4's # metadata params carry the model-specific compression ratios. self.compress_ratios = getattr(sparse_metadata_params, "compress_ratios", [1]) @@ -363,9 +559,9 @@ def __post_init__(self): # DeepSeek-V4's indexer cache uses layer-dependent compressed block sizes # (tokens_per_block // compress_ratio), so slot mappings must be built # against that stride — not kv_cache_manager.tokens_per_block directly. - tpb = self.kv_cache_manager.tokens_per_block + tpb = md.kv_cache_manager.tokens_per_block self._indexer_compress_ratio = _select_indexer_compress_ratio(self.compress_ratios) - if hasattr(self.kv_cache_manager, "compressed_block_sizes"): + if hasattr(md.kv_cache_manager, "compressed_block_sizes"): tpb = tpb // _effective_compress_ratio_divisor(self._indexer_compress_ratio) self._tokens_per_block = tpb # Mirror the indexer's two-level GVR decision. The representative @@ -374,7 +570,7 @@ def __post_init__(self): self.use_self_sampling_topk = use_self_sampling_gvr( enable_heuristic_topk=self.enable_gvr_topk, use_self_sampling_topk=sparse_metadata_params.use_self_sampling_topk, - index_topk=self.num_sparse_topk, + index_topk=md.num_sparse_topk, compress_ratio=self._indexer_compress_ratio, is_cute_dsl_available=IS_CUTLASS_DSL_AVAILABLE, sm_version=get_sm_version(), @@ -386,6 +582,7 @@ def __post_init__(self): and not self.use_self_sampling_topk ) + self._size_for_spec_decoding() self.create_buffers_for_mla_rope_append(capture_graph=capture_graph) self.create_buffers_for_indexer(capture_graph=capture_graph) self._create_nvfp4_mla_generation_buffers(capture_graph=capture_graph) @@ -393,8 +590,9 @@ def __post_init__(self): def _offloads_dsa_latent(self) -> bool: """Whether attention reads a DSA cache manager's MLA latent through the offload arena.""" - return self.sparse_metadata_params.enable_kv_cache_offload and is_dsa_cache_manager( - self.kv_cache_manager + md = self.metadata + return md.sparse_metadata_params.enable_kv_cache_offload and is_dsa_cache_manager( + md.kv_cache_manager ) def _init_dsa_sparse_offload_state(self) -> None: @@ -405,11 +603,15 @@ def _init_dsa_sparse_offload_state(self) -> None: each request's new latent row at its arena write row through them. An NVFP4 latent is dequantized for attention into the state's FP8 scratch. """ + md = self.metadata self.sparse_offload_state: Optional[SparseOffloadState] = None self.sparse_latent_block_offsets: Optional[torch.Tensor] = None self.sparse_latent_write_slots: Optional[torch.Tensor] = None self.sparse_offload_attn_indices: Optional[torch.Tensor] = None self.sparse_offload_stream: Optional[torch.cuda.Stream] = None + self.sparse_offload_start_event: Optional[torch.cuda.Event] = None + self.sparse_offload_write_event: Optional[torch.cuda.Event] = None + self.sparse_offload_fetch_event: Optional[torch.cuda.Event] = None self.sparse_offload_write_pending: Optional[int] = None self.sparse_offload_fetch_pending: Optional[int] = None self.sparse_offload_next_shared_layer: dict[int, int] = {} @@ -419,8 +621,8 @@ def _init_dsa_sparse_offload_state(self) -> None: self.sparse_offload_start_event = torch.cuda.Event() self.sparse_offload_write_event = torch.cuda.Event() self.sparse_offload_fetch_event = torch.cuda.Event() - manager = self.kv_cache_manager - layers = get_sparse_offload_metadata_layers(self) + manager = md.kv_cache_manager + layers = get_sparse_offload_metadata_layers(md) local_layers = manager.pp_layers indexer_mask = manager.indexer_k_cache_local_layer_mask self.sparse_offload_next_shared_layer = { @@ -433,12 +635,12 @@ def _init_dsa_sparse_offload_state(self) -> None: batch = manager.max_batch_size def allocate(name: str, shape: tuple[int, ...], dtype: torch.dtype) -> torch.Tensor: - return self.get_empty( - self.cuda_graph_buffers, + return md.get_empty( + md.cuda_graph_buffers, shape, dtype=dtype, cache_name=f"dsa_sparse_offload_{name}", - capture_graph=self.is_cuda_graph, + capture_graph=md.is_cuda_graph, ) # Native fetch always processes every Batch row, including for smaller @@ -465,21 +667,18 @@ def _validate_dsa_sparse_offload_batch(self) -> None: An offloading worker only decodes, one query token per request. """ + md = self.metadata self.sparse_offload_state.prepared = False - if self.num_contexts: + if md.num_contexts: raise NotImplementedError( "Sparse KV cache offload runs decode batches only; context requests reach " "an offloading worker through disaggregated KV transfer" ) - if self.seq_lens.device.type != "cpu" or self.seq_lens_kv.device.type != "cpu": + if md.seq_lens.device.type != "cpu" or md.seq_lens_kv.device.type != "cpu": raise ValueError("Sparse offload batch validation requires host sequence lengths") - if not torch.equal(self.seq_lens, self.seq_lens_kv) or not torch.all(self.seq_lens == 1): + if not torch.equal(md.seq_lens, md.seq_lens_kv) or not torch.all(md.seq_lens == 1): raise NotImplementedError("Sparse offload decode requires one query per request") - def _has_mixed_precision_staging(self) -> bool: - # No staging: attention reads the gather's FP8 scratch. - return False - @property def has_mixed_precision_high_rows(self) -> bool: """Whether this backend writes and reads a mixed-precision high pool.""" @@ -490,49 +689,170 @@ def _fill_mixed_precision_high_block_table( ) -> None: """Refresh a mixed-precision high page table over its full width; the rest stay -1. - The MTP draft loop writes past this step's kv lengths, so every column a request + The MTP draft loop writes past this step's KV lengths, so every column a request has allocated must be current. """ + md = self.metadata table = torch.full( - (self.num_seqs, high_table.shape[1]), -1, dtype=torch.int32, pin_memory=prefer_pinned() + (md.num_seqs, high_table.shape[1]), -1, dtype=torch.int32, pin_memory=prefer_pinned() ) - manager._fill_page_table(high_layer, self.request_ids, table.numpy()) - high_table[: self.num_seqs].copy_(table, non_blocking=True) + manager._fill_page_table(high_layer, md.request_ids, table.numpy()) + high_table[: md.num_seqs].copy_(table, non_blocking=True) def _fill_mixed_precision_high_block_tables(self) -> None: - for manager, high_table in ( - (self.kv_cache_manager, self.mixed_precision_high_block_table), - (self.draft_kv_cache_manager, self.draft_mixed_precision_high_block_table), - ): - if high_table is not None: - # All layers share one mixed-precision high pool, so one table serves them all. - high_layer = next(iter(manager.mixed_precision_high_layers.values())) - self._fill_mixed_precision_high_block_table(manager, high_layer, high_table) + """Refresh the mixed-precision high page table of this metadata's KV cache manager. + + A draft metadata on a separate draft KV cache manager fills its own. + """ + high_table = self.mixed_precision_high_block_table + if high_table is not None: + manager = self.metadata.kv_cache_manager + # All layers share one mixed-precision high pool, so one table serves them all. + high_layer = next(iter(manager.mixed_precision_high_layers.values())) + self._fill_mixed_precision_high_block_table(manager, high_layer, high_table) + + @property + def is_ragged_verify(self) -> bool: + return self.ragged_verify_lens is not None + + def set_gen_token_layout( + self, + *, + tokens_per_gen_step: int, + ragged_verify_lens: Optional[List[int]], + device_windows_mode: bool, + ) -> None: + """Take this step's generation-token layout; read by the next ``prepare``. + + ``tokens_per_gen_step`` is the uniform query width of a generation + request at the step's runtime draft length (0: the static draft-token + ceiling); ``ragged_verify_lens`` the per-request widths of a ragged + verify step, or None; ``device_windows_mode`` whether those widths are + only the captured shape split, the true windows arriving on device + through ``apply_device_ragged_layout`` after prepare. + """ + self.runtime_tokens_per_gen_step = tokens_per_gen_step + self.ragged_verify_lens = ragged_verify_lens + self.device_windows_mode = device_windows_mode + + def set_ragged_verify_lens(self, ragged_verify_lens: Optional[List[int]]) -> None: + """Take this step's per-request verify widths (None: a uniform step).""" + self.ragged_verify_lens = ragged_verify_lens + + def _validate_ragged_verification(self) -> None: + """Reject layouts whose stable buffers were not enabled at allocation.""" + if self.ragged_verify_lens is not None and not self.enable_ragged_verification: + raise RuntimeError( + "ragged verification metadata was installed without enabling " + "ragged-verification buffers" + ) + + @property + def gen_token_stride(self) -> int: + stride = int(self.runtime_tokens_per_gen_step) + return stride if stride > 0 else 1 + self.max_draft_tokens + + def update_indexer_kv_lens_2d(self, gen_kv_lens: torch.Tensor) -> torch.Tensor: + """Fill and return the active query-width view of persistent storage.""" + md = self.metadata + width = min(self.gen_token_stride, self.kv_lens_cuda_2d.shape[1]) + context_lens = self.kv_lens_cuda_2d[: md.num_generations, :width] + context_lens.copy_(gen_kv_lens.unsqueeze(-1).expand(-1, width)) + return context_lens + + def mla_scheduler_buffers(self, num_heads: int) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """Precompute prefixes for the same rows presented to MLA generation.""" + view = self.token_major_gen_view() + if view is None: + return None + + num_ctx = self.metadata.num_contexts + n_gen = view.num_rows - num_ctx + if not self._ragged_mla_buffers_valid: + # RoPE presents one query token per generation row. The device + # extents are causal, pre-extra-KV lengths, just like the + # cache_seq_lens scanned by the non-precomputed kernel path. + torch.cumsum( + view.prompt_lens_cuda[num_ctx:], + 0, + dtype=torch.int32, + out=self.ragged_mla_cu_q_rows[1 : n_gen + 1], + ) + self.ragged_mla_cu_q_rows[1 : n_gen + 1] *= num_heads + torch.cumsum( + view.sequence_length[num_ctx:], + 0, + dtype=torch.int32, + out=self.ragged_mla_cu_kv_seqlens[1 : n_gen + 1], + ) + self._ragged_mla_buffers_valid = True + return ( + self.ragged_mla_cu_q_rows[: n_gen + 1], + self.ragged_mla_cu_kv_seqlens[: n_gen + 1], + ) + + def gen_token_repeat_list(self) -> List[int]: + md = self.metadata + if not self.is_ragged_verify: + return [self.gen_token_stride] * md.num_generations + return list(self.ragged_verify_lens) + + def gen_token_repeats(self, device: Optional[torch.device] = None) -> torch.Tensor: + return torch.tensor(self.gen_token_repeat_list(), dtype=torch.long, device=device) + + def expand_per_gen_token(self, values: torch.Tensor, dim: int = 0) -> Tuple[torch.Tensor, int]: + """Repeat request values once per generation query token.""" + md = self.metadata + repeats = self.gen_token_repeat_list() + num_tokens = sum(repeats) + if not self.is_ragged_verify: + return values.repeat_interleave(self.gen_token_stride, dim=dim), num_tokens + if values.device == self.gen_token_repeats_cuda.device: + repeats_dev = self.gen_token_repeats_cuda[: md.num_generations] + else: + repeats_dev = torch.tensor(repeats, dtype=torch.long, device=values.device) + return ( + values.repeat_interleave(repeats_dev, dim=dim, output_size=num_tokens), + num_tokens, + ) - def prepare(self): + @property + def use_cute_dsl_paged_mqa_logits(self) -> bool: + """The Indexer's decision: the config flag gated on SM100 and cutlass-dsl.""" + md = self.metadata + return use_cute_dsl_paged_mqa_logits( + md.sparse_metadata_params.use_cute_dsl_paged_mqa_logits + ) + + def prepare(self) -> None: + md = self.metadata + self._invalidate_length_prefixes() + # Indexer.prepare below rebuilds the indexer schedule. + self.indexer_schedule_stale = False self._validate_ragged_verification() if self.sparse_offload_state is not None: + # The cache manager published the batch before the core copied its + # block offsets (prepare_batch_tables); reject what offload cannot run. self._validate_dsa_sparse_offload_batch() - self.kv_cache_manager.prepare_sparse_batch(self.request_ids) - super().prepare() self._invalidate_pool_view_cache() # Get kv lengths - assert self.kv_cache_params.use_cache is True, "DSA requires use_cache to be True" + assert md.kv_cache_params.use_cache is True, "DSA requires use_cache to be True" cached_token_lens = torch.tensor( - self.kv_cache_params.num_cached_tokens_per_seq, + md.kv_cache_params.num_cached_tokens_per_seq, dtype=torch.int, device="cpu", ) - if self.enable_helix: + helix = md.find_ext(HelixMetadata) + if helix is not None: # For Helix CP, inactive ranks only attend to previously cached # tokens (no new token appended), while active ranks add new tokens. # This mirrors the kv_lens logic in TrtllmAttentionMetadata.prepare(). - active_rank = ~self.helix_is_inactive_rank_cpu[: self.num_seqs] + active_rank = ~helix.helix_is_inactive_rank_cpu[: md.num_seqs] kv_lens = cached_token_lens.clone() - kv_lens[active_rank] += self.seq_lens_kv[active_rank] + kv_lens[active_rank] += md.seq_lens_kv[active_rank] else: - kv_lens = cached_token_lens + self.seq_lens_kv + kv_lens = cached_token_lens + md.seq_lens_kv kv_lens = self._prepare_device_window_kv_lens(kv_lens) @@ -553,123 +873,96 @@ def prepare(self): self.prepare_for_spec_decode(kv_lens) # Prepare metadata for indexer - Indexer.prepare(metadata=self) + Indexer.prepare(metadata=md) if self.sparse_offload_state is not None: - self.kv_cache_manager.prepare_sparse_offload( - self.sparse_offload_state, self.request_ids, beam_width=self.beam_width + md.kv_cache_manager.prepare_sparse_offload( + self.sparse_offload_state, md.request_ids, beam_width=md.beam_width + ) + + if md.kv_lens_offsets is not None or (helix is not None and md._helix_spec_tokens_valid): + # The core's device KV lengths carry corrections the host values + # above do not (the overlap scheduler's, or Helix's rank-local + # lengths of speculative verify groups, prepared before this + # extension); re-derive their dependents on device. + self.on_update_kv_lens() + + def adopt_target_state(self, source: "DSAMetadata") -> None: + """Start a draft from the target's indexer state, viewing its own buffers.""" + super().adopt_target_state(source) + md = self.metadata + self._invalidate_length_prefixes() + # The target's indexer schedule is current; the draft copied it. + self.indexer_schedule_stale = False + # Its pool-view cache and per-forward remap batches view its own buffers. + self._invalidate_pool_view_cache() + # Without cached-KV context MLA the full-KV maps alias the slot maps. + for alias, name in ( + ("slot_mapping_fp8_fullkv", "slot_mapping_fp8"), + ("slot_mapping_scale_fullkv", "slot_mapping_scale"), + ): + mapping = getattr(source, alias, None) + if mapping is not None and mapping is getattr(source, name): + setattr(self, alias, getattr(self, name)) + else: + setattr(self, alias, mapping) + self.kv_lens_row_reorder = ( + None + if source.kv_lens_row_reorder is None + else self.kv_lens_row_reorder_buffer[: md.num_generations] + ) + self.gen_indexer_kv_lens_cuda_runtime = None + if source.gen_indexer_kv_lens_cuda_runtime is not None: + num_contexts = md.num_contexts + self.bind_gen_indexer_kv_lens( + md.kv_lens_cuda_runtime[num_contexts : num_contexts + md.num_generations] ) def _prepare_device_window_kv_lens(self, kv_lens: torch.Tensor) -> torch.Tensor: """Bound host-side KV extents until device-selected windows are installed.""" + md = self.metadata if self.device_windows_mode and self.is_ragged_verify: - nc, ns = self.num_contexts, self.num_seqs + nc, ns = md.num_contexts, md.num_seqs kv_lens = kv_lens.clone() - kv_lens[nc:ns] += (1 + self.max_draft_tokens) - self.seq_lens_kv[nc:ns] - self.kv_lens[nc:ns] = kv_lens[nc:ns] + kv_lens[nc:ns] += (1 + self.max_draft_tokens) - md.seq_lens_kv[nc:ns] + md.kv_lens[nc:ns] = kv_lens[nc:ns] return kv_lens - def prepare_for_draft_forward(self) -> dict | None: - """Select native DSA indexer metadata for a draft forward.""" - # MTP layers on the draft manager use its mixed-precision high page table. - self.swap_for_draft( - "mixed_precision_high_block_table", self.draft_mixed_precision_high_block_table - ) - # DeepSeek-V4 metadata inherits DSA metadata, but its cache manager uses a - # different dual-pool layout. Only native DSA cache managers use the DSA - # draft-replay buffers below. - if not is_dsa_cache_manager(self.kv_cache_manager): - return None - - saved_state = { - "host_indexer_k_cache_block_offsets": self.host_indexer_k_cache_block_offsets, - "indexer_k_cache_block_offsets": self.indexer_k_cache_block_offsets, - "host_slot_mapping_fp8": self.host_slot_mapping_fp8, - "host_slot_mapping_scale": self.host_slot_mapping_scale, - "slot_mapping_fp8": self.slot_mapping_fp8, - "slot_mapping_scale": self.slot_mapping_scale, - "block_table": self.block_table, - "block_table_expanded": self.block_table_expanded, - "host_block_table_expanded": self.host_block_table_expanded, - } - # The cached-KV feature owns these references even when an optimized - # path aliases them to slot_mapping_*. With the feature disabled, the - # aliases are lazy and may not exist on the first generation replay. - if self.enable_context_mla_with_cached_kv: - saved_state.update( - { - "slot_mapping_fp8_fullkv": self.slot_mapping_fp8_fullkv, - "slot_mapping_scale_fullkv": self.slot_mapping_scale_fullkv, - } - ) - - # Rebind to the draft manager's dedicated buffers instead of - # overwriting the target tensors in place. Rebinding is invisible to - # CUDA graph capture, so the target and draft segments of the graph - # bake distinct addresses (like draft_kv_cache_block_offsets) and no - # graph-recorded copy from a transient host buffer is needed. - self.swap_for_draft( - "host_indexer_k_cache_block_offsets", self.host_draft_indexer_k_cache_block_offsets - ) - self.swap_for_draft( - "indexer_k_cache_block_offsets", self.draft_indexer_k_cache_block_offsets - ) - self.swap_for_draft("host_slot_mapping_fp8", self.host_draft_slot_mapping_fp8) - self.swap_for_draft("slot_mapping_fp8", self.draft_slot_mapping_fp8) - self.swap_for_draft("host_slot_mapping_scale", self.host_draft_slot_mapping_scale) - self.swap_for_draft("slot_mapping_scale", self.draft_slot_mapping_scale) - self.swap_for_draft("block_table", self.draft_block_table) - self.swap_for_draft("block_table_expanded", self.draft_block_table_expanded) - self.swap_for_draft("host_block_table_expanded", self.host_draft_block_table_expanded) - # recompute_context_kv_gather_mappings() rebinds the aliases below. - self.record_draft_swap("slot_mapping_fp8_fullkv") - self.record_draft_swap("slot_mapping_scale_fullkv") - self._invalidate_pool_view_cache() - - # Recording a capture executes no kernels, so the draft mappings only - # need refreshing when the transfers actually run: eager forwards - # (warmup) and the pre-replay call from model_engine. The per-step - # advance inside the captured graph re-derives slot mappings on - # device from the rebound block-offset buffer. - # kv_cache_manager was already swapped to the draft manager above. - if not torch.cuda.is_current_stream_capturing(): - self.prepare_for_indexer_k_cache() - self._refresh_expanded_block_table() - Indexer.recompute_slot_mappings(self) - Indexer.recompute_context_kv_gather_mappings(self) - - return saved_state - - def restore_after_draft_forward(self, saved_state: dict | None) -> None: - """Restore native DSA indexer metadata after a draft forward.""" - if saved_state is None: - return - - # restore_draft_swaps() has already rebound the swapped fields. - self._invalidate_pool_view_cache() - if "slot_mapping_fp8_fullkv" not in saved_state: - # Without cached-KV context MLA the aliases track the target mappings. - self.slot_mapping_fp8_fullkv = self.slot_mapping_fp8 - self.slot_mapping_scale_fullkv = self.slot_mapping_scale - def get_indexer_kv_lens(self, kv_lens: torch.Tensor) -> torch.Tensor: if self._indexer_compress_ratio <= 1: return kv_lens return kv_lens // self._indexer_compress_ratio + def bind_gen_indexer_kv_lens(self, gen_kv_lens: torch.Tensor) -> torch.Tensor: + """Bind and return the generation rows' indexer KV lengths. + + ``gen_kv_lens`` are the rows' device KV lengths. The result keeps its + address across steps (a view of them, or of a persistent buffer), so + CUDA graphs read the lengths each ``prepare`` writes. + """ + if self._indexer_compress_ratio <= 1: + lens = gen_kv_lens + else: + lens = self._gen_indexer_kv_lens_buffer[: gen_kv_lens.shape[0]] + torch.floor_divide(gen_kv_lens, self._indexer_compress_ratio, out=lens) + self.gen_indexer_kv_lens_cuda_runtime = lens + return lens + def get_indexer_max_seq_len(self) -> int: + md = self.metadata if self._indexer_compress_ratio <= 1: - return self.kv_cache_manager.max_seq_len - return max(1, self.kv_cache_manager.max_seq_len // self._indexer_compress_ratio) + return md.kv_cache_manager.max_seq_len + return max(1, md.kv_cache_manager.max_seq_len // self._indexer_compress_ratio) def warmup_cute_dsl_radix_topk(self, next_n: int) -> None: """Pre-compile CuTe DSL radix variants not covered by engine warmup.""" - sparse_params = self.sparse_metadata_params + md = self.metadata + sparse_params = md.sparse_metadata_params if not self.use_cute_dsl_topk or ( sparse_params.enable_heuristic_topk and get_sm_version() >= 100 ): return - if self.kv_cache_manager is None: + if md.kv_cache_manager is None: return top_k = self.sparse_mla_topk if not top_k: @@ -709,9 +1002,10 @@ def warmup_selfsampling_topk( """ # same two-level dispatch and hardware gates as the indexer __init__: # never compile these kernels on unsupported stacks during warmup + md = self.metadata if not IS_CUTLASS_DSL_AVAILABLE or get_sm_version() not in (100, 103): return - if not self.enable_gvr_topk or self.kv_cache_manager is None: + if not self.enable_gvr_topk or md.kv_cache_manager is None: return if not self.use_self_sampling_topk: return @@ -741,7 +1035,7 @@ def warmup_selfsampling_topk( # forwarded only when enabled (keeps the default warmup calls unchanged) det_kw = ( {"deterministic": True} - if getattr(self.sparse_metadata_params, "gvr_deterministic_topk", False) + if getattr(md.sparse_metadata_params, "gvr_deterministic_topk", False) else {} ) try: @@ -783,25 +1077,44 @@ def warmup_selfsampling_topk( "will JIT-compile lazily on first touch instead." ) + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + # The draft loop changed the draft's query and KV lengths; the draft + # loop itself sets any other shape of the step (no context cached KV). + md = self.metadata + self._invalidate_length_prefixes() + if md.num_generations > 0: + self.max_gen_seq_len = md.seq_lens[md.num_contexts : md.num_seqs].max().item() + else: + self.max_gen_seq_len = 0 + # Every draft step after the first of an index-sharing draft loop + # reuses the TopK selection the first step captured, so the indexer + # schedule is not read before the next prepare rebuilds it. + skip_indexer_schedule = bool( + self.index_share.in_mtp_draft_loop and self.sparse_metadata_params.mtp_index_share + ) + self.on_update_kv_lens(skip_indexer_schedule=skip_indexer_schedule) + + def _invalidate_length_prefixes(self) -> None: + """Mark the layer-invariant length prefixes stale; the next MLA layer rebuilds them.""" + self._ragged_mla_buffers_valid = False + def on_update_kv_lens(self, skip_indexer_schedule: bool = False) -> None: - # After changing the kv_lens/kv_lens_cuda, we may need to update other metadatas. - # Especially for the changes in DecoderInputPreparer.preprocess_inputs(). - # - # NOTE: - # In overlap scheduler + speculative decoding, kv_lens_cuda can be corrected at runtime - # (inside preprocess_inputs) to account for variable accepted tokens. The indexer - # slot_mapping_* buffers also depend on these effective cached lengths. If we do not - # refresh slot mappings here, indexer K-cache updates can be written with stale offsets. - # - # skip_indexer_schedule leaves the indexer's MQA-logits schedule as is; see - # indexer_schedule_stale for what that covers and when it is legal. - self.indexer_schedule_stale = skip_indexer_schedule + """Re-derive the state that depends on the device KV lengths, on device. - super().on_update_kv_lens() + For lengths that differ from the host values ``prepare`` derived + from: the overlap scheduler's correction (``kv_lens_offsets``), which + accounts for the accepted draft tokens, and a draft step's advance. + The indexer slot mappings depend on these cached lengths too; stale + ones would write indexer K-cache entries at the wrong offsets. - # preprocess_inputs() also uses this as a general hook to "invalidate per-forward-pass - # caches so they are recomputed (and captured) on every _forward_step". Invalidate the - # pool_view cache here so it is recomputed on the next + ``skip_indexer_schedule`` leaves the indexer's MQA-logits schedule and + GVR row order as they are and marks them stale + (``indexer_schedule_stale``). + """ + md = self.metadata + self.indexer_schedule_stale = skip_indexer_schedule + # The pool views are rebuilt by the next # transform_local_topk_and_prepare_pool_view() call. self._invalidate_pool_view_cache() @@ -818,45 +1131,48 @@ def on_update_kv_lens(self, skip_indexer_schedule: bool = False) -> None: # it stale; those configs keep the eager chain. fused_eligible = ( _fused_dsa_meta_enabled() - and self.kv_cache_manager is not None - and self.num_tokens > 0 - and self.num_generations > 0 - and self.num_contexts == 0 + and md.kv_cache_manager is not None + and md.num_tokens > 0 + and md.num_generations > 0 + and md.num_contexts == 0 and not self.use_fp8_ds_mla + and not self.is_ragged_verify ) # The draft loop can rewrite seq_lens after prepare() built this map. - if self.num_tokens > 0 and not fused_eligible: - self.req_idx_per_token[: self.num_tokens] = build_req_idx_per_token( - self.seq_lens_cuda[: self.num_seqs], self.num_tokens - ).to(self.req_idx_per_token.dtype) - - # The overlap correction may have changed request KV lengths after - # prepare(); rebuild each ragged row's causal extent on device. + # Under ragged verification, prepare_for_indices_conversion() or + # apply_device_ragged_layout() already populated req_idx_per_token + # (including valid request indices for bucket-padded tokens). + if md.num_tokens > 0 and not fused_eligible and not self.is_ragged_verify: + self.index_share.req_idx_per_token[: md.num_tokens] = build_req_idx_per_token( + md.seq_lens_cuda[: md.num_seqs], md.num_tokens + ).to(self.index_share.req_idx_per_token.dtype) + + # Rebuild each ragged row's causal extent from the device KV lengths. if self.enable_ragged_verification: self.refresh_ragged_row_kv_lens() self.refresh_token_major_gen_rows() # Uncached DSA prefill aliases these scatter maps as gather maps. - if self.kv_cache_manager is not None and self.num_tokens > 0 and not fused_eligible: - seq_lens = self.seq_lens_cuda[: self.num_seqs] + if md.kv_cache_manager is not None and md.num_tokens > 0 and not fused_eligible: + seq_lens = md.seq_lens_cuda[: md.num_seqs] # Runtime cached lengths after overlap/spec-dec correction. - start_positions = self.kv_lens_cuda[: self.num_seqs] - seq_lens + start_positions = md.kv_lens_cuda[: md.num_seqs] - seq_lens - req_indices = self.req_idx_per_token[: self.num_tokens].to(dtype=torch.int64) + req_indices = self.index_share.req_idx_per_token[: md.num_tokens].to(dtype=torch.int64) seq_starts = torch.cumsum(seq_lens, dim=0, dtype=torch.int64) - seq_lens.to(torch.int64) token_offsets = ( - torch.arange(self.num_tokens, device=seq_lens.device, dtype=torch.int64) + torch.arange(md.num_tokens, device=seq_lens.device, dtype=torch.int64) - seq_starts[req_indices] ) global_positions = start_positions[req_indices] + token_offsets if self.use_fp8_ds_mla: - self.token_positions_cuda[: self.num_tokens] = global_positions.to(torch.int32) + self.token_positions_cuda[: md.num_tokens] = global_positions.to(torch.int32) # Honor MXFP4 indexer K cache layout (½ byte per value vs FP8's # 1 byte) when the cache manager exposes a use_fp4 flag. - index_head_dim = self.kv_cache_manager.index_head_dim - use_fp4 = getattr(self.kv_cache_manager, "use_fp4", False) + index_head_dim = md.kv_cache_manager.index_head_dim + use_fp4 = getattr(md.kv_cache_manager, "use_fp4", False) data_bytes_per_token = index_head_dim // 2 if use_fp4 else index_head_dim fp8_indices, scale_indices = _compute_slot_mappings( global_positions, @@ -867,51 +1183,49 @@ def on_update_kv_lens(self, skip_indexer_schedule: bool = False) -> None: self.indexer_quant_block_size, data_bytes_per_token=data_bytes_per_token, ) - self.slot_mapping_fp8[: self.num_tokens] = fp8_indices - self.slot_mapping_scale[: self.num_tokens] = scale_indices + self.slot_mapping_fp8[: md.num_tokens] = fp8_indices + self.slot_mapping_scale[: md.num_tokens] = scale_indices - if self.num_generations > 0: + if md.num_generations > 0: if not fused_eligible: torch.cumsum( - self.kv_lens_cuda[ - self.num_contexts : self.num_seqs - ], # num_contexts should be 0 + md.kv_lens_cuda[md.num_contexts : md.num_seqs], # num_contexts should be 0 dim=0, dtype=torch.int64, - out=self.gen_kv_indptr[1 : self.num_generations + 1], + out=self.gen_kv_indptr[1 : md.num_generations + 1], ) torch.cumsum( ( - self.kv_lens_cuda[self.num_contexts : self.num_seqs] - - self.seq_lens_cuda[self.num_contexts : self.num_seqs] + md.kv_lens_cuda[md.num_contexts : md.num_seqs] + - md.seq_lens_cuda[md.num_contexts : md.num_seqs] ), dim=0, dtype=torch.int64, - out=self.gen_cached_token_indptr[1 : self.num_generations + 1], + out=self.gen_cached_token_indptr[1 : md.num_generations + 1], ) - if self.num_generations > 0 and not skip_indexer_schedule: + if md.num_generations > 0 and not skip_indexer_schedule: self._rebuild_indexer_schedule() if fused_eligible: if not getattr(self, "_fused_dsa_meta_armed", False): logger.info( "Fused DSA decode-metadata " - f"kernel armed (num_seqs={self.num_seqs}, " - f"num_tokens={self.num_tokens})." + f"kernel armed (num_seqs={md.num_seqs}, " + f"num_tokens={md.num_tokens})." ) self._fused_dsa_meta_armed = True self._run_fused_dsa_decode_metadata() if not skip_indexer_schedule: self._compute_kv_lens_row_reorder() - self.prepare_dense_topk_indices(self.kv_lens_cuda, device=True) + self.prepare_dense_topk_indices(md.kv_lens_cuda, device=True) def _rebuild_indexer_schedule(self) -> None: """Rebuild what the indexer's MQA-logits + TopK pass reads from the gen KV lengths: the DeepGEMM schedule(s), the 2D/expanded indexer KV lengths and their runtime copy.""" - gen_kv_lens = self.kv_lens_cuda[self.num_contexts : self.num_seqs] - gen_indexer_kv_lens = self.get_indexer_kv_lens(gen_kv_lens) - self.gen_indexer_kv_lens_cuda_runtime = gen_indexer_kv_lens + md = self.metadata + gen_kv_lens = md.kv_lens_cuda[md.num_contexts : md.num_seqs] + gen_indexer_kv_lens = self.bind_gen_indexer_kv_lens(gen_kv_lens) scheduler_context_lens = self.update_indexer_kv_lens_2d(gen_indexer_kv_lens) scheduler_metadata_buffer = get_paged_mqa_logits_metadata( gen_indexer_kv_lens.view(-1, 1), _DG_SCHEDULE_BLOCK_KV, self.num_sms @@ -938,55 +1252,56 @@ def _rebuild_indexer_schedule(self) -> None: scheduler_metadata_buffer_expanded, non_blocking=True ) if self.expand_for_dsl and self.dsl_kernel_atoms > 1: - num_rows = self.num_generations * self.dsl_kernel_atoms + num_rows = md.num_generations * self.dsl_kernel_atoms atom_lens = gen_indexer_kv_lens.unsqueeze(1) - self.dsl_atom_ctx_offsets self.kv_lens_expanded_cuda[:num_rows].copy_(atom_lens.reshape(-1)) - scheduler_metadata_buffer_expanded = get_paged_mqa_logits_metadata( - self.kv_lens_expanded_cuda[:num_rows].view(-1, 1), - _DG_SCHEDULE_BLOCK_KV, - self.num_sms, - ) - self.scheduler_metadata_buffer_expanded.copy_( - scheduler_metadata_buffer_expanded, non_blocking=True - ) + self._schedule_dsl_expanded_rows(num_rows) if self.expand_for_dsl and self.dsl_expand_factor > 1: expand_factor = self.dsl_expand_factor - num_tokens = self.num_generations * expand_factor + num_tokens = md.num_generations * expand_factor gen_kv_lens_expanded = gen_indexer_kv_lens.repeat_interleave(expand_factor) self.kv_lens_expanded_cuda[:num_tokens].copy_(gen_kv_lens_expanded) - scheduler_metadata_buffer_expanded = get_paged_mqa_logits_metadata( - self.kv_lens_expanded_cuda[:num_tokens].view(-1, 1), - _DG_SCHEDULE_BLOCK_KV, - self.num_sms, - ) - self.scheduler_metadata_buffer_expanded.copy_( - scheduler_metadata_buffer_expanded, non_blocking=True - ) + self._schedule_dsl_expanded_rows(num_tokens) + + def _schedule_dsl_expanded_rows(self, num_rows: int) -> None: + """Build the DeepGEMM schedule of the CuTe DSL indexer's expanded rows. + + Reads the first ``num_rows`` entries of ``kv_lens_expanded_cuda``. + """ + scheduler_metadata_buffer_expanded = get_paged_mqa_logits_metadata( + self.kv_lens_expanded_cuda[:num_rows].view(-1, 1), + _DG_SCHEDULE_BLOCK_KV, + self.num_sms, + ) + self.scheduler_metadata_buffer_expanded.copy_( + scheduler_metadata_buffer_expanded, non_blocking=True + ) def _run_fused_dsa_decode_metadata(self): """Fill req_idx_per_token + slot mappings + gen indptrs via one Triton launch (see kernels.py), replacing the eager chain for the pure-decode step. Capture-safe: the launch is pre-compiled at warmup and replays inside the decode CUDA graph.""" - num_tokens = self.num_tokens - num_seqs = self.num_seqs + md = self.metadata + num_tokens = md.num_tokens + num_seqs = md.num_seqs # The fused header writes gen_*_indptr[1 : num_seqs + 1] while the eager # chain writes [1 : num_generations + 1]; they coincide only because the # eligibility gate requires num_contexts == 0 (=> num_seqs == # num_generations). Pin that coupling down explicitly. - assert num_seqs == self.num_generations, ( + assert num_seqs == md.num_generations, ( f"fused DSA metadata expects num_seqs ({num_seqs}) == " - f"num_generations ({self.num_generations})" + f"num_generations ({md.num_generations})" ) - index_head_dim = self.kv_cache_manager.index_head_dim - use_fp4 = getattr(self.kv_cache_manager, "use_fp4", False) + index_head_dim = md.kv_cache_manager.index_head_dim + use_fp4 = getattr(md.kv_cache_manager, "use_fp4", False) data_bytes_per_token = index_head_dim // 2 if use_fp4 else index_head_dim fused_dsa_decode_metadata( - self.seq_lens_cuda[:num_seqs], - self.kv_lens_cuda[:num_seqs], + md.seq_lens_cuda[:num_seqs], + md.kv_lens_cuda[:num_seqs], self.indexer_k_cache_block_offsets[:num_seqs], - self.req_idx_per_token[:num_tokens], + self.index_share.req_idx_per_token[:num_tokens], self.slot_mapping_fp8[:num_tokens], self.slot_mapping_scale[:num_tokens], self.gen_kv_indptr[: num_seqs + 1], @@ -1001,63 +1316,27 @@ def _run_fused_dsa_decode_metadata(self): def _compute_kv_lens_row_reorder(self) -> None: """Prepare the longest-job-first GVR row order once per forward step.""" + md = self.metadata next_n = self.gen_token_stride - if self.needs_gvr_prior and self.num_generations * next_n >= 2 * self.num_sms: - gen_kv_lens = self.kv_lens_cuda[self.num_contexts : self.num_seqs] + if self.needs_gvr_prior and md.num_generations * next_n >= 2 * self.num_sms: + gen_kv_lens = md.kv_lens_cuda[md.num_contexts : md.num_seqs] order = torch.argsort(gen_kv_lens, descending=True).to(torch.int32) - self.kv_lens_row_reorder_buffer[: self.num_generations].copy_(order) - self.kv_lens_row_reorder = self.kv_lens_row_reorder_buffer[: self.num_generations] + self.kv_lens_row_reorder_buffer[: md.num_generations].copy_(order) + self.kv_lens_row_reorder = self.kv_lens_row_reorder_buffer[: md.num_generations] else: self.kv_lens_row_reorder = None - def update_for_spec_dec(self): - super().update_for_spec_dec() - # host - self.max_ctx_kv_len = 0 - self.num_ctx_cached_tokens = 0 - self.max_gen_seq_len = 1 - - # device. Inside an index-sharing draft loop every call here prepares either a - # TopK-reuse draft step or the post-loop target forward, which rebuilds in - # _preprocess_inputs: the indexer schedule is never read before its next full rebuild. - skip_indexer_schedule = ( - self.in_mtp_draft_loop and self.sparse_metadata_params.mtp_index_share - ) - self.on_update_kv_lens(skip_indexer_schedule=skip_indexer_schedule) - # Create buffers for mla_rope_append_paged_kv_assign_q def create_buffers_for_mla_rope_append(self, capture_graph=False): # New context buffers for dsa - if not self.enable_context_mla_with_cached_kv: - self.ctx_cached_token_indptr = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_requests + 1,), - cache_name="ctx_cached_token_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_cached_token_indptr = torch.zeros_like( - self.ctx_cached_token_indptr, - device="cpu", - pin_memory=prefer_pinned(), - ) - self.ctx_kv_indptr = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_requests + 1,), - cache_name="ctx_kv_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_kv_indptr = torch.zeros_like( - self.ctx_kv_indptr, - device="cpu", - pin_memory=prefer_pinned(), - ) + md = self.metadata + if not md.enable_context_mla_with_cached_kv: + md.allocate_context_kv_indptrs(md.cuda_graph_buffers, capture_graph) # New generation buffers for dsa - self.gen_cached_token_indptr = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_requests + 1,), + self.gen_cached_token_indptr = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_requests + 1,), cache_name="gen_cached_token_indptr", dtype=torch.int64, capture_graph=capture_graph, @@ -1067,9 +1346,9 @@ def create_buffers_for_mla_rope_append(self, capture_graph=False): device="cpu", pin_memory=prefer_pinned(), ) - self.gen_kv_indptr = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_requests + 1,), + self.gen_kv_indptr = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_requests + 1,), cache_name="gen_kv_indptr", dtype=torch.int64, capture_graph=capture_graph, @@ -1082,9 +1361,10 @@ def create_buffers_for_mla_rope_append(self, capture_graph=False): def _create_nvfp4_mla_generation_buffers(self, capture_graph=False): """Allocate stable NVFP4 gather outputs for all MTP generation rows.""" + md = self.metadata self.nvfp4_mla_fp8_scratch = None self.nvfp4_mla_compact_indices = None - if getattr(self.kv_cache_manager, "dtype", None) != tensorrt_llm.bindings.DataType.NVFP4: + if getattr(md.kv_cache_manager, "dtype", None) != tensorrt_llm.bindings.DataType.NVFP4: return if self._offloads_dsa_latent(): # Offloaded rows are dequantized into the offload state's scratch instead. @@ -1095,38 +1375,39 @@ def _create_nvfp4_mla_generation_buffers(self, capture_graph=False): # so the dequantized FP8 rows and compact offsets must cover the full # verification window. The buffers have stable addresses for CUDA # graphs and are reused sequentially by every attention layer. - max_gen_tokens = self.max_num_sequences * (1 + self.max_draft_tokens) - self.nvfp4_mla_fp8_scratch = self.get_empty( - self.cuda_graph_buffers, + max_gen_tokens = md.max_num_sequences * (1 + self.max_draft_tokens) + self.nvfp4_mla_fp8_scratch = md.get_empty( + md.cuda_graph_buffers, ( max_gen_tokens, - self.num_sparse_topk, - self.kv_cache_manager.head_dim, + md.num_sparse_topk, + md.kv_cache_manager.head_dim, ), cache_name="nvfp4_mla_fp8_scratch", dtype=torch.float8_e4m3fn, capture_graph=capture_graph, ) - self.nvfp4_mla_compact_indices = self.get_empty( - self.cuda_graph_buffers, - (max_gen_tokens, self.num_sparse_topk), + self.nvfp4_mla_compact_indices = md.get_empty( + md.cuda_graph_buffers, + (max_gen_tokens, md.num_sparse_topk), cache_name="nvfp4_mla_compact_indices", dtype=torch.int32, capture_graph=capture_graph, ) def _create_radix_aux_buffers(self, capture_graph: bool) -> None: - max_num_rows = self.max_num_sequences * self._radix_rows_per_sequence - radix_shape = (max_num_rows, _MAX_RADIX_BLOCKS_PER_ROW, self.num_sparse_topk) - self.radix_aux_indices = self.get_empty( - self.cuda_graph_buffers, + md = self.metadata + max_num_rows = md.max_num_sequences * self._radix_rows_per_sequence + radix_shape = (max_num_rows, _MAX_RADIX_BLOCKS_PER_ROW, md.num_sparse_topk) + self.radix_aux_indices = md.get_empty( + md.cuda_graph_buffers, radix_shape, cache_name="radix_aux_indices", dtype=torch.int32, capture_graph=capture_graph, ) - self.radix_aux_logits = self.get_empty( - self.cuda_graph_buffers, + self.radix_aux_logits = md.get_empty( + md.cuda_graph_buffers, radix_shape, cache_name="radix_aux_logits", dtype=torch.float32, @@ -1134,12 +1415,13 @@ def _create_radix_aux_buffers(self, capture_graph: bool) -> None: ) def create_buffers_for_indexer(self, capture_graph=False): - sparse_metadata_params = self.sparse_metadata_params + md = self.metadata + sparse_metadata_params = md.sparse_metadata_params if not isinstance(sparse_metadata_params, DSAMetadataParams): raise ValueError("DSA sparse attention metadata params are not set") - self.indexer_k_cache_block_offsets = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.kv_cache_manager.max_blocks_per_seq], + self.indexer_k_cache_block_offsets = md.get_empty( + md.cuda_graph_buffers, + [md.max_num_sequences, md.kv_cache_manager.max_blocks_per_seq], cache_name="indexer_k_cache_block_offsets", dtype=torch.int32, capture_graph=capture_graph, @@ -1154,26 +1436,23 @@ def create_buffers_for_indexer(self, capture_graph=False): sparse_metadata_params.has_shared_indexer_layers or sparse_metadata_params.mtp_index_share ): - self.shared_topk_indices = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens, self.num_sparse_topk), - cache_name="shared_topk_indices", - dtype=torch.int32, - capture_graph=capture_graph, + self.index_share.bind_buffers( + shared_topk_indices=md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens, md.num_sparse_topk), + cache_name="shared_topk_indices", + dtype=torch.int32, + capture_graph=capture_graph, + ) ) else: - self.shared_topk_indices = None - - self.indexer_skip_topk = False - self.in_mtp_draft_loop = False - self.mtp_num_accepted = None - self.indexer_schedule_stale = False + self.index_share.bind_buffers(shared_topk_indices=None) # Indexer metadata # Separate slot mappings for non-interleaved layout (flat byte indices) - self.slot_mapping_fp8 = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.slot_mapping_fp8 = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="slot_mapping_fp8", dtype=torch.int64, capture_graph=capture_graph, @@ -1183,9 +1462,9 @@ def create_buffers_for_indexer(self, capture_graph=False): device="cpu", pin_memory=prefer_pinned(), ) - self.slot_mapping_scale = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.slot_mapping_scale = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="slot_mapping_scale", dtype=torch.int64, capture_graph=capture_graph, @@ -1197,110 +1476,45 @@ def create_buffers_for_indexer(self, capture_graph=False): ) self.token_positions_cuda = None if self.use_fp8_ds_mla: - self.token_positions_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.token_positions_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="token_positions_cuda", dtype=torch.int32, capture_graph=capture_graph, ) - # Allocate separate indexer block-offset and slot-mapping buffers for - # the draft KV cache manager, mirroring draft_kv_cache_block_offsets: - # the draft-replay context swaps these in by rebinding, so CUDA graph - # capture bakes distinct addresses for the target and draft segments - # and both sides can be refreshed eagerly outside the graph. - if is_dsa_cache_manager(self.draft_kv_cache_manager): - self.draft_indexer_k_cache_block_offsets = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.draft_kv_cache_manager.max_blocks_per_seq], - cache_name="draft_indexer_k_cache_block_offsets", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.host_draft_indexer_k_cache_block_offsets = torch.zeros_like( - self.draft_indexer_k_cache_block_offsets, - device="cpu", - pin_memory=prefer_pinned(), - ) - self.draft_slot_mapping_fp8 = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), - cache_name="draft_slot_mapping_fp8", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_draft_slot_mapping_fp8 = torch.zeros_like( - self.draft_slot_mapping_fp8, - device="cpu", - pin_memory=prefer_pinned(), - ) - self.draft_slot_mapping_scale = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), - cache_name="draft_slot_mapping_scale", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_draft_slot_mapping_scale = torch.zeros_like( - self.draft_slot_mapping_scale, - device="cpu", - pin_memory=prefer_pinned(), - ) # Only when MLA chunked prefill is enabled, we need to gather the full KV for indexer's logit computation. # Allocate these buffers dynamically in Indexer.prepare() # based on the actual total_kv_len to save memory. - if self.enable_context_mla_with_cached_kv: + if md.enable_context_mla_with_cached_kv: self.slot_mapping_fp8_fullkv = None self.slot_mapping_scale_fullkv = None # Per-token request index buffer for topk_indices conversion - self.req_idx_per_token = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), - cache_name="req_idx_per_token", - dtype=torch.int32, - capture_graph=capture_graph, + self.index_share.bind_buffers( + req_idx_per_token=md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), + cache_name="req_idx_per_token", + dtype=torch.int32, + capture_graph=capture_graph, + ) ) self.host_req_idx_per_token = torch.empty_like( - self.req_idx_per_token, device="cpu", pin_memory=prefer_pinned() + self.index_share.req_idx_per_token, device="cpu", pin_memory=prefer_pinned() ) # Block table for topk_indices conversion (shared for context and generation) - self.block_table = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.kv_cache_manager.max_blocks_per_seq], - cache_name="block_table", - dtype=torch.int32, - capture_graph=capture_graph, - ) - # Mixed-precision high page table; -1 = not in the high pool. - if getattr(self.kv_cache_manager, "mixed_precision_high_layers", None): - self.mixed_precision_high_block_table = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.kv_cache_manager.max_blocks_per_seq], - cache_name="mixed_precision_high_block_table", + self.index_share.bind_buffers( + block_table=md.get_empty( + md.cuda_graph_buffers, + [md.max_num_sequences, md.kv_cache_manager.max_blocks_per_seq], + cache_name="block_table", dtype=torch.int32, capture_graph=capture_graph, ) - if getattr(self.draft_kv_cache_manager, "mixed_precision_high_layers", None): - self.draft_mixed_precision_high_block_table = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.draft_kv_cache_manager.max_blocks_per_seq], - cache_name="draft_mixed_precision_high_block_table", - dtype=torch.int32, - capture_graph=capture_graph, - ) - if is_dsa_cache_manager(self.draft_kv_cache_manager): - self.draft_block_table = self.get_empty( - self.cuda_graph_buffers, - [ - self.max_num_sequences, - self.draft_kv_cache_manager.max_blocks_per_seq, - ], - cache_name="draft_block_table", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.scheduler_metadata_buffer = self.get_empty( - self.cuda_graph_buffers, + ) + self._allocate_mixed_precision_high_block_table(capture_graph) + self.scheduler_metadata_buffer = md.get_empty( + md.cuda_graph_buffers, (self.num_sms + 1, 2), cache_name="scheduler_metadata_buffer", dtype=torch.int32, @@ -1315,8 +1529,8 @@ def create_buffers_for_indexer(self, capture_graph=False): # path keeps using `scheduler_metadata_buffer`. Always allocate (a # few KB) so transitions in `update_spec_dec_param` don't have to # special-case its existence. - self.scheduler_metadata_buffer_full_next_n = self.get_empty( - self.cuda_graph_buffers, + self.scheduler_metadata_buffer_full_next_n = md.get_empty( + md.cuda_graph_buffers, (self.num_sms + 1, 2), cache_name="scheduler_metadata_buffer_full_next_n", dtype=torch.int32, @@ -1328,25 +1542,25 @@ def create_buffers_for_indexer(self, capture_graph=False): # slice per forward. Avoids per-forward .expand().contiguous() # allocations that would break CUDA graphs. self._create_kv_lens_2d_buffer(capture_graph=capture_graph) - self.cu_seqlen_ks = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.cu_seqlen_ks = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="cu_seqlen_ks", dtype=torch.int32, capture_graph=capture_graph, ) - self.cu_seqlen_ke = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens,), + self.cu_seqlen_ke = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens,), cache_name="cu_seqlen_ke", dtype=torch.int32, capture_graph=capture_graph, ) # Topk indices buffer to support skip indexer for requests with short sequence lengths if self.enable_indexer_skip: - self.topk_indices_buffer = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_tokens, self.num_sparse_topk), + self.topk_indices_buffer = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_tokens, md.num_sparse_topk), cache_name="topk_indices_buffer", dtype=torch.int32, capture_graph=capture_graph, @@ -1357,21 +1571,21 @@ def create_buffers_for_indexer(self, capture_graph=False): pin_memory=prefer_pinned(), ) if self.needs_gvr_prior: - self.gvr_prior_indices = self.get_empty( - self.cuda_graph_buffers, + self.gvr_prior_indices = md.get_empty( + md.cuda_graph_buffers, ( - self.kv_cache_manager.num_local_layers, - self.max_num_sequences, - self.num_sparse_topk, + md.kv_cache_manager.num_local_layers, + md.max_num_sequences, + md.num_sparse_topk, ), cache_name="gvr_prior_indices", dtype=torch.int32, capture_graph=capture_graph, ) self.gvr_prior_indices.zero_() - self.kv_lens_row_reorder_buffer = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + self.kv_lens_row_reorder_buffer = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), cache_name="kv_lens_row_reorder_buffer", dtype=torch.int32, capture_graph=capture_graph, @@ -1388,27 +1602,54 @@ def _draft_sizing_cap(self) -> int: return self.max_draft_tokens return max(int(cap), self.max_draft_tokens) + def _allocate_mixed_precision_high_block_table(self, capture_graph: bool) -> None: + """Allocate the mixed-precision high page table when the KV cache manager has a high pool.""" + md = self.metadata + if getattr(md.kv_cache_manager, "mixed_precision_high_layers", None): + self.mixed_precision_high_block_table = md.get_empty( + md.cuda_graph_buffers, + [md.max_num_sequences, md.kv_cache_manager.max_blocks_per_seq], + cache_name="mixed_precision_high_block_table", + dtype=torch.int32, + capture_graph=capture_graph, + ) + def _create_kv_lens_2d_buffer(self, capture_graph=False): - """Pre-allocated buffer for the DeepGEMM 2D context_lens API. + """Pre-allocated buffers for the generation rows' indexer KV lengths. - Preparation and on_update_kv_lens fill the persistent backing storage + ``kv_lens_cuda_2d`` serves the DeepGEMM 2D context_lens API. + Preparation and the device refresh fill the persistent backing storage in place. Scheduling receives a contiguous view at the active runtime query width, which may be smaller than the allocation ceiling. """ - self.kv_lens_cuda_2d = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences, 1 + self._draft_sizing_cap), + md = self.metadata + self.kv_lens_cuda_2d = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences, 1 + self._draft_sizing_cap), cache_name="kv_lens_cuda_2d", dtype=torch.int32, capture_graph=capture_graph, ) + # Compressed indexer KV lengths; uncompressed ones view the core's. + self._gen_indexer_kv_lens_buffer = ( + md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), + cache_name="gen_indexer_kv_lens", + dtype=torch.int32, + capture_graph=capture_graph, + ) + if self._indexer_compress_ratio > 1 + else None + ) # TODO: remove these expanded buffers when fp8_paged_mqa_logits supports an arbitrary number of MTP draft tokens. def create_expanded_buffers(self, capture_graph=False): """Create expanded KV-length and block-table buffers for speculative decoding.""" - self.kv_lens_expanded_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences * (1 + self._draft_sizing_cap),), + md = self.metadata + self.kv_lens_expanded_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences * (1 + self._draft_sizing_cap),), cache_name="kv_lens_expanded_cuda", dtype=torch.int32, capture_graph=capture_graph, @@ -1442,11 +1683,11 @@ def create_expanded_buffers(self, capture_graph=False): self.ragged_mla_cu_kv_seqlens = None if self.enable_ragged_verification: self._create_ragged_verification_buffers(capture_graph) - self.block_table_expanded = self.get_empty( - self.cuda_graph_buffers, + self.block_table_expanded = md.get_empty( + md.cuda_graph_buffers, [ - self.max_num_sequences * (1 + self._draft_sizing_cap), - self.kv_cache_manager.max_blocks_per_seq, + md.max_num_sequences * (1 + self._draft_sizing_cap), + md.kv_cache_manager.max_blocks_per_seq, ], cache_name="block_table_expanded", dtype=torch.int32, @@ -1457,24 +1698,8 @@ def create_expanded_buffers(self, capture_graph=False): device="cpu", pin_memory=prefer_pinned(), ) - if is_dsa_cache_manager(self.draft_kv_cache_manager): - self.draft_block_table_expanded = self.get_empty( - self.cuda_graph_buffers, - [ - self.max_num_sequences * (1 + self._draft_sizing_cap), - self.draft_kv_cache_manager.max_blocks_per_seq, - ], - cache_name="draft_block_table_expanded", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.host_draft_block_table_expanded = torch.zeros_like( - self.draft_block_table_expanded, - device="cpu", - pin_memory=prefer_pinned(), - ) - self.scheduler_metadata_buffer_expanded = self.get_empty( - self.cuda_graph_buffers, + self.scheduler_metadata_buffer_expanded = md.get_empty( + md.cuda_graph_buffers, (self.num_sms + 1, 2), cache_name="scheduler_metadata_buffer_expanded", dtype=torch.int32, @@ -1484,9 +1709,10 @@ def create_expanded_buffers(self, capture_graph=False): def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: """Allocate persistent row layouts required only by ragged verification.""" # Stable-address repeat vector used by in-graph ragged expansions. - self.gen_token_repeats_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), + md = self.metadata + self.gen_token_repeats_cuda = md.get_empty( + md.cuda_graph_buffers, + (md.max_num_sequences,), cache_name="gen_token_repeats_cuda", dtype=torch.int64, capture_graph=capture_graph, @@ -1496,10 +1722,10 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: device="cpu", pin_memory=prefer_pinned(), ) - row_cap = self.max_num_sequences * (1 + self._draft_sizing_cap) + row_cap = md.max_num_sequences * (1 + self._draft_sizing_cap) for name in ("ragged_mla_cu_q_rows", "ragged_mla_cu_kv_seqlens"): - prefix = self.get_empty( - self.cuda_graph_buffers, + prefix = md.get_empty( + md.cuda_graph_buffers, (row_cap + 1,), cache_name=name, dtype=torch.int32, @@ -1507,8 +1733,8 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: ) prefix[:1].zero_() setattr(self, name, prefix) - self.row_kv_lens_cuda = self.get_empty( - self.cuda_graph_buffers, + self.row_kv_lens_cuda = md.get_empty( + md.cuda_graph_buffers, (row_cap,), cache_name="row_kv_lens_cuda", dtype=torch.int32, @@ -1517,8 +1743,8 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: self.row_kv_lens_host = torch.zeros_like( self.row_kv_lens_cuda, device="cpu", pin_memory=prefer_pinned() ) - self.row_kv_correction_cuda = self.get_empty( - self.cuda_graph_buffers, + self.row_kv_correction_cuda = md.get_empty( + md.cuda_graph_buffers, (row_cap,), cache_name="row_kv_correction_cuda", dtype=torch.int32, @@ -1527,8 +1753,8 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: self.row_kv_correction_host = torch.zeros_like( self.row_kv_correction_cuda, device="cpu", pin_memory=prefer_pinned() ) - self.row_req_idx_cuda = self.get_empty( - self.cuda_graph_buffers, + self.row_req_idx_cuda = md.get_empty( + md.cuda_graph_buffers, (row_cap,), cache_name="row_req_idx_cuda", dtype=torch.long, @@ -1540,23 +1766,23 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: # MLA RoPE and sparse-MLA generation consume one row per query token; # every other consumer keeps using request-major metadata. - attn_rows_cap = self.max_num_sequences * (2 + self._draft_sizing_cap) - self.attn_row_kv_lens_cuda = self.get_empty( - self.cuda_graph_buffers, + attn_rows_cap = md.max_num_sequences * (2 + self._draft_sizing_cap) + self.attn_row_kv_lens_cuda = md.get_empty( + md.cuda_graph_buffers, (attn_rows_cap,), cache_name="attn_row_kv_lens_cuda", dtype=torch.int32, capture_graph=capture_graph, ) - self.attn_row_kv_correction_cuda = self.get_empty( - self.cuda_graph_buffers, + self.attn_row_kv_correction_cuda = md.get_empty( + md.cuda_graph_buffers, (attn_rows_cap,), cache_name="attn_row_kv_correction_cuda", dtype=torch.int32, capture_graph=capture_graph, ) - self.attn_row_req_idx_cuda = self.get_empty( - self.cuda_graph_buffers, + self.attn_row_req_idx_cuda = md.get_empty( + md.cuda_graph_buffers, (attn_rows_cap,), cache_name="attn_row_req_idx_cuda", dtype=torch.long, @@ -1574,8 +1800,8 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: self.attn_row_request_types_host = torch.ones( (attn_rows_cap,), dtype=torch.int32, pin_memory=prefer_pinned() ) - self.attn_row_prompt_lens_cuda = self.get_empty( - self.cuda_graph_buffers, + self.attn_row_prompt_lens_cuda = md.get_empty( + md.cuda_graph_buffers, (attn_rows_cap,), cache_name="attn_row_prompt_lens_cuda", dtype=torch.int32, @@ -1585,17 +1811,17 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: (attn_rows_cap,), dtype=torch.int32, pin_memory=prefer_pinned() ) num_attention_op_pools = getattr( - self.kv_cache_manager, + md.kv_cache_manager, "num_attention_op_pools", - self.kv_cache_manager.num_pools, + md.kv_cache_manager.num_pools, ) - self.attn_row_block_offsets = self.get_empty( - self.cuda_graph_buffers, + self.attn_row_block_offsets = md.get_empty( + md.cuda_graph_buffers, ( num_attention_op_pools, attn_rows_cap, 2, - self.kv_cache_manager.max_blocks_per_seq, + md.kv_cache_manager.max_blocks_per_seq, ), cache_name="attn_row_block_offsets", dtype=torch.int32, @@ -1604,87 +1830,44 @@ def _create_ragged_verification_buffers(self, capture_graph: bool) -> None: # This function is only used to create the expanded buffers when the max_draft_tokens is changed. # TODO: remove this function once fp8_paged_mqa_logits supports an arbitrary number of MTP draft tokens. - def update_spec_dec_param( - self, - batch_size, - is_spec_decoding_enabled, - is_spec_dec_tree, - is_spec_dec_dynamic_tree, - max_draft_len, - max_total_draft_tokens, - spec_metadata: Optional["SpecMetadata"] = None, - spec_tree_manager: Optional["SpecTreeManager"] = None, - num_contexts: int = 0, - ): - """Update speculative decoding parameters and create expanded buffers.""" - super().update_spec_dec_param( - batch_size, - is_spec_decoding_enabled, - is_spec_dec_tree, - is_spec_dec_dynamic_tree, - max_draft_len, - max_total_draft_tokens, - spec_metadata, - spec_tree_manager, - num_contexts=num_contexts, - ) - self.max_draft_tokens = max_draft_len - rows_per_sequence = max(1 + max_draft_len, 1 + (max_total_draft_tokens or 0)) - if is_spec_dec_dynamic_tree and spec_tree_manager is not None: - rows_per_sequence = max(rows_per_sequence, spec_tree_manager._internal_buf_dim) - resize_radix_workspace = rows_per_sequence != self._radix_rows_per_sequence + def _size_for_spec_decoding(self) -> None: + """Size the draft-token buffers from the metadata's spec-dec settings.""" + md = self.metadata + spec = md.find_ext(SpecDecodingMetadata) + if spec is None: + return + config = spec.spec_decoding + self.max_draft_tokens = config.max_draft_len + rows_per_sequence = max(1 + config.max_draft_len, 1 + config.max_total_draft_tokens) + if config.is_spec_dec_dynamic_tree: + rows_per_sequence = max(rows_per_sequence, config.tree_buffer_dim) self._radix_rows_per_sequence = rows_per_sequence - self._draft_alloc_cap = max( - int(max_total_draft_tokens), - int(getattr(self, "_draft_alloc_cap", 0) or 0), - ) - capture_graph = self.is_cuda_graph - max_gen_tokens = self.max_num_sequences * (1 + self.max_draft_tokens) - if ( - self.nvfp4_mla_fp8_scratch is not None - and self.nvfp4_mla_fp8_scratch.shape[0] != max_gen_tokens - ): - self._create_nvfp4_mla_generation_buffers(capture_graph=capture_graph) - previous_cap = int(getattr(self, "_expanded_alloc_draft_cap", -1)) - allocation_cap = max(self._draft_sizing_cap, previous_cap) - if allocation_cap > previous_cap: - self._expanded_alloc_draft_cap = allocation_cap - self._create_kv_lens_2d_buffer(capture_graph=capture_graph) - self.create_expanded_buffers(capture_graph=capture_graph) - if resize_radix_workspace: - self._create_radix_aux_buffers(capture_graph) + self._draft_alloc_cap = int(config.max_total_draft_tokens) + self._expanded_alloc_draft_cap = self._draft_sizing_cap def _update_indexer_k_cache_block_offsets(self) -> torch.Tensor: """Refresh INDEX_KEY offsets and return their physical pool slots.""" - cache_manager = self.kv_cache_manager + md = self.metadata + cache_manager = md.kv_cache_manager # Raw block IDs can exceed the primary pool after host-cache offload; # the manager resolves their current physical slots. pool_indices = cache_manager.get_pool_block_indices( - self.num_seqs, - request_ids=self.request_ids, - num_contexts=self.num_contexts, - beam_width=self.beam_width, + md.num_seqs, + request_ids=md.request_ids, + num_contexts=md.num_contexts, + beam_width=md.beam_width, ) page_indices = pool_indices * cache_manager.indexer_k_cache_page_scale num_blocks = page_indices.shape[1] - self.host_indexer_k_cache_block_offsets[: self.num_seqs, :num_blocks].copy_(page_indices) - self.indexer_k_cache_block_offsets[: self.num_seqs].copy_( - self.host_indexer_k_cache_block_offsets[: self.num_seqs], non_blocking=True + self.host_indexer_k_cache_block_offsets[: md.num_seqs, :num_blocks].copy_(page_indices) + self.indexer_k_cache_block_offsets[: md.num_seqs].copy_( + self.host_indexer_k_cache_block_offsets[: md.num_seqs], non_blocking=True ) # Sanitize graph-padding entries that may be stale after cache # eviction or host-cache onboarding. self.indexer_k_cache_block_offsets.clamp_(min=0) return pool_indices - def set_skip_topk(self, skip: bool) -> None: - self.indexer_skip_topk = skip - - def set_in_mtp_draft_loop(self, active: bool) -> None: - self.in_mtp_draft_loop = active - - def set_mtp_num_accepted(self, num_accepted: Optional[torch.Tensor]) -> None: - self.mtp_num_accepted = num_accepted - def _invalidate_pool_view_cache(self): """Invalidate the cached pool view and related step-invariant values. @@ -1710,14 +1893,15 @@ def _ensure_pool_view_cached(self): Python/CUDA overhead per layer. Safety: _invalidate_pool_view_cache() is called unconditionally at the - start of every step (prepare() and on_update_kv_lens()), so the boolean + start of every step (prepare() and the device refresh), so the boolean flag is always cleared before the first per-layer call within a step. """ - if self._pool_cache_valid and self._cached_kv_mgr_id == id(self.kv_cache_manager): + md = self.metadata + if self._pool_cache_valid and self._cached_kv_mgr_id == id(md.kv_cache_manager): return - pool = self.kv_cache_manager.get_unique_primary_pool() - kv_cache_manager = self.kv_cache_manager + pool = md.kv_cache_manager.get_unique_primary_pool() + kv_cache_manager = md.kv_cache_manager num_blocks, num_layers = pool.shape[:2] self._cached_tokens_per_block = kv_cache_manager.tokens_per_block head_dim = kv_cache_manager.head_dim @@ -1729,11 +1913,11 @@ def _ensure_pool_view_cached(self): self._cached_pool_view = None else: self._cached_pool_view = pool.squeeze(2).view(-1, 1, head_dim) - self._cached_block_table_ctx = self.block_table[: self.num_contexts] - self._cached_block_table_gen = self.block_table[self.num_contexts : self.num_seqs] - self._cached_req_idx_ctx = self.req_idx_per_token[: self.num_ctx_tokens] + self._cached_block_table_ctx = self.index_share.block_table[: md.num_contexts] + self._cached_block_table_gen = self.index_share.block_table[md.num_contexts : md.num_seqs] + self._cached_req_idx_ctx = self.index_share.req_idx_per_token[: md.num_ctx_tokens] self._cached_req_idx_gen = ( - self.req_idx_per_token[self.num_ctx_tokens : self.num_tokens] - self.num_contexts + self.index_share.req_idx_per_token[md.num_ctx_tokens : md.num_tokens] - md.num_contexts ) self._cached_kv_mgr_id = id(kv_cache_manager) self._pool_cache_valid = True @@ -1754,7 +1938,8 @@ def _ensure_group_remap_struct(self): stride_factor). Singleton / non-uniform groups fall back to the per-layer single-op path (identical to flag-off behavior). """ - kvm = self.kv_cache_manager + md = self.metadata + kvm = md.kv_cache_manager if self._group_remap_struct is not None and self._group_remap_struct_kv_id == id(kvm): return self._group_remap_struct @@ -1793,10 +1978,10 @@ def _ensure_group_remap_struct(self): # Device for the small per-group layer-offset tensors. device = None - if self.shared_topk_indices is not None: - device = self.shared_topk_indices.device - elif self.block_table is not None: - device = self.block_table.device + if self.index_share.shared_topk_indices is not None: + device = self.index_share.shared_topk_indices.device + elif self.index_share.block_table is not None: + device = self.index_share.block_table.device group_active = {} group_scale = {} @@ -1828,6 +2013,7 @@ def _ensure_group_remap_struct(self): @maybe_compile(dynamic=True) def _get_dense_topk_indices(self, seq_lens, kv_lens, num_tokens): + md = self.metadata device = kv_lens.device past_kv_lens = kv_lens - seq_lens # get position ids @@ -1839,50 +2025,51 @@ def _get_dense_topk_indices(self, seq_lens, kv_lens, num_tokens): repeated_offsets = per_seq_offsets[batch_indices] position_ids = global_indices + repeated_offsets # get the dense topk indices with causal mask - range_row = torch.arange(self.num_sparse_topk, device=device) + range_row = torch.arange(md.num_sparse_topk, device=device) mask = range_row <= position_ids.unsqueeze(1) return torch.where(mask, range_row, -1) def prepare_dense_topk_indices(self, kv_lens, device=False): # device=False means use CPU """Prepare dense TopK indices for short sequences that skip the indexer.""" - if self.num_contexts > 0 and self.skip_indexer_for_ctx_reqs: - ctx_range = slice(self.num_ctx_tokens) + md = self.metadata + if md.num_contexts > 0 and self.skip_indexer_for_ctx_reqs: + ctx_range = slice(md.num_ctx_tokens) if device: self.topk_indices_buffer[ctx_range, :].copy_( self._get_dense_topk_indices( - self.seq_lens_cuda[: self.num_contexts], - kv_lens[: self.num_contexts], - self.num_ctx_tokens, + md.seq_lens_cuda[: md.num_contexts], + kv_lens[: md.num_contexts], + md.num_ctx_tokens, ), non_blocking=True, ) else: self.host_topk_indices_buffer[ctx_range, :] = self._get_dense_topk_indices( - self.seq_lens[: self.num_contexts], - kv_lens[: self.num_contexts], - self.num_ctx_tokens, + md.seq_lens[: md.num_contexts], + kv_lens[: md.num_contexts], + md.num_ctx_tokens, ) self.topk_indices_buffer[ctx_range, :].copy_( self.host_topk_indices_buffer[ctx_range, :], non_blocking=True ) - if self.num_generations > 0 and self.skip_indexer_for_gen_reqs: - gen_range = slice(self.num_ctx_tokens, self.num_tokens) + if md.num_generations > 0 and self.skip_indexer_for_gen_reqs: + gen_range = slice(md.num_ctx_tokens, md.num_tokens) if device: self.topk_indices_buffer[gen_range, :].copy_( self._get_dense_topk_indices( - self.seq_lens_cuda[self.num_contexts : self.num_seqs], - kv_lens[self.num_contexts : self.num_seqs], - self.num_tokens - self.num_ctx_tokens, + md.seq_lens_cuda[md.num_contexts : md.num_seqs], + kv_lens[md.num_contexts : md.num_seqs], + md.num_tokens - md.num_ctx_tokens, ), non_blocking=True, ) else: self.host_topk_indices_buffer[gen_range, :] = self._get_dense_topk_indices( - self.seq_lens[self.num_contexts : self.num_seqs], - kv_lens[self.num_contexts : self.num_seqs], - self.num_tokens - self.num_ctx_tokens, + md.seq_lens[md.num_contexts : md.num_seqs], + kv_lens[md.num_contexts : md.num_seqs], + md.num_tokens - md.num_ctx_tokens, ) self.topk_indices_buffer[gen_range, :].copy_( self.host_topk_indices_buffer[gen_range, :], non_blocking=True @@ -1898,6 +2085,7 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): # TODO: # - Drop this sm90 branch (and the expanded buffers) once # fp8_paged_mqa_logits supports an arbitrary next_n on sm90 too. + md = self.metadata use_dsl = self.use_cute_dsl_paged_mqa_logits if not self.is_ragged_verify: self._ragged_num_rows = 0 @@ -1906,7 +2094,7 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): # The DSL paged-MQA path reconstructs rows from one scalar window # and cannot express a per-request split. use_dsl = False - assert not self.is_spec_decoding_enabled, ( + assert not md.is_spec_decoding_enabled, ( "ragged verification requires dense spec-decoding metadata to be disabled" ) @@ -1915,7 +2103,7 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): ) if self.use_expanded_buffers_for_mtp: if self.is_ragged_verify: - n_gen = self.num_generations + n_gen = md.num_generations self.host_gen_token_repeats[:n_gen].copy_( torch.tensor(self.gen_token_repeat_list(), dtype=torch.int64) ) @@ -1923,17 +2111,17 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): self.host_gen_token_repeats[:n_gen], non_blocking=True ) - gen_kv_lens = self.get_indexer_kv_lens(kv_lens[self.num_contexts : self.num_seqs]) + gen_kv_lens = self.get_indexer_kv_lens(kv_lens[md.num_contexts : md.num_seqs]) gen_kv_lens_expanded, num_tokens = self.expand_per_gen_token(gen_kv_lens) self.kv_lens_expanded_host[:num_tokens].copy_(gen_kv_lens_expanded) self.kv_lens_expanded_cuda[:num_tokens].copy_( self.kv_lens_expanded_host[:num_tokens], non_blocking=True ) - if self.kv_cache_manager is not None and self.num_generations > 0: + if md.kv_cache_manager is not None and md.num_generations > 0: max_len = self.host_indexer_k_cache_block_offsets.shape[1] gen_blocks = self.host_indexer_k_cache_block_offsets[ - self.num_contexts : self.num_seqs, :max_len + md.num_contexts : md.num_seqs, :max_len ] expanded_blocks, _ = self.expand_per_gen_token(gen_blocks, dim=0) self.host_block_table_expanded[:num_tokens, :max_len].copy_(expanded_blocks) @@ -1947,16 +2135,16 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): self._prepare_token_major_gen_rows(kv_lens) self.expand_for_dsl = ( - use_dsl and self.kv_cache_manager is not None and self.max_draft_tokens >= 1 + use_dsl and md.kv_cache_manager is not None and self.max_draft_tokens >= 1 ) - if self.expand_for_dsl and self.num_generations > 0: + if self.expand_for_dsl and md.num_generations > 0: next_n = self.gen_token_stride - kernel_atoms = (1, 2, 3) if self.kv_cache_manager.use_fp4 else (1, 2, 3, 4) - gen_kv_lens = self.get_indexer_kv_lens(kv_lens[self.num_contexts : self.num_seqs]) + kernel_atoms = (1, 2, 3) if md.kv_cache_manager.use_fp4 else (1, 2, 3, 4) + gen_kv_lens = self.get_indexer_kv_lens(kv_lens[md.num_contexts : md.num_seqs]) max_ctx = int(gen_kv_lens.max().item()) if gen_kv_lens.numel() else 0 expand_factor, atom, kernel_split_atoms = _pick_dsl_split( next_n, - batch_size=self.num_generations, + batch_size=md.num_generations, max_ctx=max_ctx, num_sms=self.num_sms, kernel_atoms=kernel_atoms, @@ -1968,12 +2156,12 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): if kernel_split_atoms > 1: offsets = _atom_ctx_offsets(next_n, kernel_split_atoms) if self.dsl_atom_ctx_offsets_key != (next_n, kernel_split_atoms): - # Built outside graph capture; on_update_kv_lens only subtracts. + # Built outside graph capture; the device refresh only subtracts. self.dsl_atom_ctx_offsets = torch.tensor( offsets, dtype=torch.int32, device="cuda" ) self.dsl_atom_ctx_offsets_key = (next_n, kernel_split_atoms) - num_rows = self.num_generations * kernel_split_atoms + num_rows = md.num_generations * kernel_split_atoms atom_lens = gen_kv_lens.unsqueeze(1) - torch.tensor( offsets, dtype=gen_kv_lens.dtype, device=gen_kv_lens.device ) @@ -1982,13 +2170,15 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): self.kv_lens_expanded_cuda[:num_rows].copy_( self.kv_lens_expanded_host[:num_rows], non_blocking=True ) + self._schedule_dsl_expanded_rows(num_rows) if expand_factor > 1: - num_tokens = self.num_generations * expand_factor + num_tokens = md.num_generations * expand_factor gen_kv_lens_expanded = gen_kv_lens.repeat_interleave(expand_factor) self.kv_lens_expanded_host[:num_tokens].copy_(gen_kv_lens_expanded) self.kv_lens_expanded_cuda[:num_tokens].copy_( self.kv_lens_expanded_host[:num_tokens], non_blocking=True ) + self._schedule_dsl_expanded_rows(num_tokens) self._refresh_expanded_block_table(expand_factor) else: self.dsl_expand_factor = 1 @@ -1997,14 +2187,15 @@ def prepare_for_spec_decode(self, kv_lens: torch.Tensor): def _prepare_ragged_row_kv_lens(self, kv_lens: torch.Tensor) -> None: """Populate the causal KV extent of each ragged generation row.""" + md = self.metadata verify_lens = self.ragged_verify_lens if not verify_lens: return - gen_kv_lens = kv_lens[self.num_contexts : self.num_seqs].tolist() + gen_kv_lens = kv_lens[md.num_contexts : md.num_seqs].tolist() assert len(gen_kv_lens) == len(verify_lens), ( f"ragged verify lengths {len(verify_lens)} != generation requests {len(gen_kv_lens)}" ) - expected_gen_tokens = self.num_tokens - self.num_ctx_tokens + expected_gen_tokens = md.num_tokens - md.num_ctx_tokens assert sum(verify_lens) == expected_gen_tokens, ( f"ragged verify windows sum to {sum(verify_lens)} but this step has " f"{expected_gen_tokens} generation tokens" @@ -2034,10 +2225,11 @@ def _prepare_ragged_row_kv_lens(self, kv_lens: torch.Tensor) -> None: def refresh_ragged_row_kv_lens(self) -> None: """Refresh ragged extents after overlap changes ``kv_lens_cuda``.""" + md = self.metadata num_rows = self._ragged_num_rows if not self.is_ragged_verify or num_rows <= 0: return - gen_kv_lens = self.kv_lens_cuda[self.num_contexts : self.num_seqs] + gen_kv_lens = md.kv_lens_cuda[md.num_contexts : md.num_seqs] row_kv_lens = self.row_kv_lens_cuda[:num_rows] torch.index_select( gen_kv_lens, @@ -2049,13 +2241,14 @@ def refresh_ragged_row_kv_lens(self) -> None: def _prepare_token_major_gen_rows(self, kv_lens: torch.Tensor) -> None: """Build the temporary one-row-per-query-token attention view.""" + md = self.metadata verify_lens = self.ragged_verify_lens num_gen_rows = self._ragged_num_rows if not verify_lens or num_gen_rows <= 0: self._attn_num_rows = 0 return - num_contexts = self.num_contexts + num_contexts = md.num_contexts num_rows = num_contexts + num_gen_rows capacity = self.attn_row_kv_lens_cuda.shape[0] assert num_rows <= capacity, ( @@ -2069,7 +2262,7 @@ def _prepare_token_major_gen_rows(self, kv_lens: torch.Tensor) -> None: torch.arange(num_contexts, dtype=torch.long) ) self.attn_row_prompt_lens_cpu[:num_contexts].copy_( - self.prompt_lens_cpu[:num_contexts].to(torch.int32) + md.prompt_lens_cpu[:num_contexts].to(torch.int32) ) self.attn_row_kv_lens_host[num_contexts:num_rows].copy_( @@ -2105,11 +2298,12 @@ def _prepare_token_major_gen_rows(self, kv_lens: torch.Tensor) -> None: def refresh_token_major_gen_rows(self) -> None: """Refresh token-major KV extents after overlap correction.""" + md = self.metadata num_rows = self._attn_num_rows if not self.is_ragged_verify or num_rows <= 0: return torch.index_select( - self.kv_lens_cuda[: self.num_seqs], + md.kv_lens_cuda[: md.num_seqs], 0, self.attn_row_req_idx_cuda[:num_rows], out=self.attn_row_kv_lens_cuda[:num_rows], @@ -2118,10 +2312,11 @@ def refresh_token_major_gen_rows(self) -> None: def refresh_token_major_block_table(self) -> None: """Expand the main attention block table along its sequence axis.""" + md = self.metadata num_rows = self._attn_num_rows - if num_rows <= 0 or self.kv_cache_block_offsets is None: + if num_rows <= 0 or md.kv_cache_block_offsets is None: return - source = self.kv_cache_block_offsets + source = md.kv_cache_block_offsets num_pools = min(source.shape[0], self.attn_row_block_offsets.shape[0]) width = min(source.shape[-1], self.attn_row_block_offsets.shape[-1]) torch.index_select( @@ -2161,15 +2356,24 @@ def apply_device_ragged_layout( verify_lens: torch.Tensor, req_idx: torch.Tensor, kv_correction: torch.Tensor, + kv_lens_delta: Optional[torch.Tensor] = None, ) -> None: """Install device-selected windows into stable captured buffers.""" + md = self.metadata self._validate_ragged_verification() if not self.is_ragged_verify: raise RuntimeError("device ragged layout requires enabled ragged verification") - num_contexts = self.num_contexts + num_contexts = md.num_contexts num_rows = self._ragged_num_rows - num_generations = self.num_generations - self.seq_lens_cuda[num_contexts : self.num_seqs] = verify_lens.to(self.seq_lens_cuda.dtype) + num_generations = md.num_generations + md.set_seq_lens( + slice(num_contexts, md.num_seqs), device=verify_lens.to(md.seq_lens_cuda.dtype) + ) + if kv_lens_delta is not None: + md.adjust_kv_lens( + slice(num_contexts, num_contexts + kv_lens_delta.shape[0]), kv_lens_delta + ) + md.resync_length_derived_state() self.gen_token_repeats_cuda[:num_generations] = verify_lens.to(torch.int64) self.row_req_idx_cuda[:num_rows] = req_idx self.row_kv_correction_cuda[:num_rows] = kv_correction @@ -2177,10 +2381,10 @@ def apply_device_ragged_layout( attn_row_end = attn_row_start + num_rows self.attn_row_req_idx_cuda[attn_row_start:attn_row_end] = req_idx + num_contexts self.attn_row_kv_correction_cuda[attn_row_start:attn_row_end] = kv_correction - token_start = self.num_ctx_tokens + token_start = md.num_ctx_tokens token_end = token_start + num_rows - self.req_idx_per_token[token_start:token_end] = (req_idx + num_contexts).to( - self.req_idx_per_token.dtype + self.index_share.req_idx_per_token[token_start:token_end] = (req_idx + num_contexts).to( + self.index_share.req_idx_per_token.dtype ) self.refresh_token_major_block_table() width = min( @@ -2188,14 +2392,18 @@ def apply_device_ragged_layout( self.block_table_expanded.shape[-1], ) self.block_table_expanded[:num_rows, :width] = ( - self.indexer_k_cache_block_offsets[num_contexts : self.num_seqs, :width] + self.indexer_k_cache_block_offsets[num_contexts : md.num_seqs, :width] .index_select(0, req_idx) .clamp_(min=0) ) + if kv_lens_delta is not None: + self.on_update_kv_lens() + md.clear_lengths_changed() def _refresh_expanded_block_table(self, repeat_factor: Optional[int] = None): """Refresh the active cache's expanded INDEX_KEY page table.""" - if self.kv_cache_manager is None or self.num_generations == 0: + md = self.metadata + if md.kv_cache_manager is None or md.num_generations == 0: return use_runtime_layout = False if repeat_factor is None: @@ -2208,12 +2416,12 @@ def _refresh_expanded_block_table(self, repeat_factor: Optional[int] = None): max_len = self.host_indexer_k_cache_block_offsets.shape[1] gen_block_tensor = self.host_indexer_k_cache_block_offsets[ - self.num_contexts : self.num_seqs, :max_len + md.num_contexts : md.num_seqs, :max_len ] if use_runtime_layout: expanded_blocks, num_tokens = self.expand_per_gen_token(gen_block_tensor, dim=0) else: - num_tokens = self.num_generations * repeat_factor + num_tokens = md.num_generations * repeat_factor expanded_blocks = gen_block_tensor.repeat_interleave(repeat_factor, dim=0) self.host_block_table_expanded[:num_tokens, :max_len].copy_( expanded_blocks, non_blocking=True @@ -2224,26 +2432,28 @@ def _refresh_expanded_block_table(self, repeat_factor: Optional[int] = None): self.block_table_expanded.clamp_(min=0) def prepare_for_indexer_k_cache(self): - if self.kv_cache_manager is None: + md = self.metadata + if md.kv_cache_manager is None: return # Keep physical slots for the primary-pool TopK conversion below. pool_indices = self._update_indexer_k_cache_block_offsets() # Build block_table for topk_indices conversion (actual block allocation) cached_token_lens = torch.tensor( - self.kv_cache_params.num_cached_tokens_per_seq, + md.kv_cache_params.num_cached_tokens_per_seq, dtype=torch.int, device="cpu", ) - if self.enable_helix: - active_rank = ~self.helix_is_inactive_rank_cpu[: self.num_seqs] + helix = md.find_ext(HelixMetadata) + if helix is not None: + active_rank = ~helix.helix_is_inactive_rank_cpu[: md.num_seqs] kv_lens = cached_token_lens.clone() - kv_lens[active_rank] += self.seq_lens_kv[active_rank] + kv_lens[active_rank] += md.seq_lens_kv[active_rank] else: - kv_lens = cached_token_lens + self.seq_lens_kv - tokens_per_block = self.kv_cache_manager.tokens_per_block - num_blocks_per_seq = (kv_lens[: self.num_seqs] + tokens_per_block - 1) // tokens_per_block - max_blocks_used = num_blocks_per_seq.max().item() if self.num_seqs > 0 else 1 + kv_lens = cached_token_lens + md.seq_lens_kv + tokens_per_block = md.kv_cache_manager.tokens_per_block + num_blocks_per_seq = (kv_lens[: md.num_seqs] + tokens_per_block - 1) // tokens_per_block + max_blocks_used = num_blocks_per_seq.max().item() if md.num_seqs > 0 else 1 # pool_indices already has correct values; set padding to -1. # Stage through a fresh pinned buffer: an async H2D from pageable # memory would block the host behind the busy execution stream. @@ -2255,89 +2465,65 @@ def prepare_for_indexer_k_cache(self): host_block_table.copy_(pool_indices[:, :max_blocks_used]) pad_cols = torch.arange(max_blocks_used, dtype=num_blocks_per_seq.dtype) host_block_table.masked_fill_( - pad_cols.unsqueeze(0) >= num_blocks_per_seq[: self.num_seqs].unsqueeze(1), -1 + pad_cols.unsqueeze(0) >= num_blocks_per_seq[: md.num_seqs].unsqueeze(1), -1 ) # Copy to GPU - self.block_table[: self.num_seqs, :max_blocks_used].copy_( + self.index_share.block_table[: md.num_seqs, :max_blocks_used].copy_( host_block_table, non_blocking=True ) def prepare_for_skip_indexer(self, kv_lens: torch.Tensor): - num_extra_kv_tokens = self.kv_cache_params.num_extra_kv_tokens - if self.num_contexts > 0 and self.enable_indexer_skip: + md = self.metadata + num_extra_kv_tokens = md.kv_cache_params.num_extra_kv_tokens + if md.num_contexts > 0 and self.enable_indexer_skip: # Minus the number of extra KV tokens because when using one-model MTP, the # draft layers needs more KV tokens for the next draft forwards. self.skip_indexer_for_ctx_reqs = ( - kv_lens[: self.num_contexts].max().item() - <= self.num_sparse_topk - num_extra_kv_tokens + kv_lens[: md.num_contexts].max().item() <= md.num_sparse_topk - num_extra_kv_tokens ) else: self.skip_indexer_for_ctx_reqs = False - if self.num_generations > 0 and self.enable_indexer_skip: + if md.num_generations > 0 and self.enable_indexer_skip: # Minus the number of extra KV tokens because when using one-model MTP, the # draft layers needs more KV tokens for the next draft forwards. self.skip_indexer_for_gen_reqs = ( - kv_lens[self.num_contexts : self.num_seqs].max().item() - <= self.num_sparse_topk - num_extra_kv_tokens + kv_lens[md.num_contexts : md.num_seqs].max().item() + <= md.num_sparse_topk - num_extra_kv_tokens ) else: self.skip_indexer_for_gen_reqs = False self.prepare_dense_topk_indices(kv_lens) def prepare_for_mla_rope_append(self, cached_token_lens: torch.Tensor, kv_lens: torch.Tensor): - if self.num_contexts > 0: - self.num_ctx_cached_tokens = cached_token_lens[: self.num_contexts].sum().item() - self.num_ctx_mla_kv_tokens = kv_lens[: self.num_contexts].sum().item() - self.max_ctx_kv_len = kv_lens[: self.num_contexts].max().item() - self.max_ctx_seq_len = self.seq_lens[: self.num_contexts].max().item() - # context cached token indptr - torch.cumsum( - cached_token_lens[: self.num_contexts], - dim=0, - dtype=torch.int64, - out=self.host_ctx_cached_token_indptr[1 : self.num_contexts + 1], - ) - self.ctx_cached_token_indptr[: self.num_contexts + 1].copy_( - self.host_ctx_cached_token_indptr[: self.num_contexts + 1], non_blocking=True - ) - # context kv indptr - torch.cumsum( - kv_lens[: self.num_contexts], - dim=0, - dtype=torch.int64, - out=self.host_ctx_kv_indptr[1 : self.num_contexts + 1], - ) - self.ctx_kv_indptr[: self.num_contexts + 1].copy_( - self.host_ctx_kv_indptr[: self.num_contexts + 1], non_blocking=True - ) + md = self.metadata + md.stage_context_cached_kv(cached_token_lens, kv_lens) + if md.num_contexts > 0: + self.num_ctx_mla_kv_tokens = kv_lens[: md.num_contexts].sum().item() else: - self.num_ctx_cached_tokens = 0 self.num_ctx_mla_kv_tokens = 0 - self.max_ctx_kv_len = 0 - self.max_ctx_seq_len = 0 - if self.num_generations > 0: - self.max_gen_seq_len = self.seq_lens[self.num_contexts : self.num_seqs].max().item() + if md.num_generations > 0: + self.max_gen_seq_len = md.seq_lens[md.num_contexts : md.num_seqs].max().item() # generation cached token indptr torch.cumsum( - cached_token_lens[self.num_contexts : self.num_seqs], + cached_token_lens[md.num_contexts : md.num_seqs], dim=0, dtype=torch.int64, - out=self.host_gen_cached_token_indptr[1 : self.num_generations + 1], + out=self.host_gen_cached_token_indptr[1 : md.num_generations + 1], ) - self.gen_cached_token_indptr[: self.num_generations + 1].copy_( - self.host_gen_cached_token_indptr[: self.num_generations + 1], non_blocking=True + self.gen_cached_token_indptr[: md.num_generations + 1].copy_( + self.host_gen_cached_token_indptr[: md.num_generations + 1], non_blocking=True ) # generation kv indptr torch.cumsum( - kv_lens[self.num_contexts : self.num_seqs], + kv_lens[md.num_contexts : md.num_seqs], dim=0, dtype=torch.int64, - out=self.host_gen_kv_indptr[1 : self.num_generations + 1], + out=self.host_gen_kv_indptr[1 : md.num_generations + 1], ) - self.gen_kv_indptr[: self.num_generations + 1].copy_( - self.host_gen_kv_indptr[: self.num_generations + 1], non_blocking=True + self.gen_kv_indptr[: md.num_generations + 1].copy_( + self.host_gen_kv_indptr[: md.num_generations + 1], non_blocking=True ) else: self.max_gen_seq_len = 0 @@ -2345,12 +2531,19 @@ def prepare_for_mla_rope_append(self, cached_token_lens: torch.Tensor, kv_lens: def prepare_for_indices_conversion(self): # Build req_idx_per_token for topk_indices conversion # Use pinned staging buffer to avoid pageable H2D memcpy - self.host_req_idx_per_token[: self.num_tokens] = torch.repeat_interleave( - torch.arange(self.num_seqs, dtype=torch.int32), - self.seq_lens, + md = self.metadata + self.host_req_idx_per_token[: md.num_tokens] = torch.repeat_interleave( + torch.arange(md.num_seqs, dtype=torch.int32), + md.seq_lens, dim=0, ) - self.req_idx_per_token[: self.num_tokens].copy_( - self.host_req_idx_per_token[: self.num_tokens], + self.index_share.req_idx_per_token[: md.num_tokens].copy_( + self.host_req_idx_per_token[: md.num_tokens], non_blocking=True, ) + + +class DSAtrtllmAttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the DSA extension attached.""" + + backend_extension_types = (DSAMetadata,) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/module.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/module.py index b1f2962e6c2f..ca2156fca9dd 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/module.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/module.py @@ -474,9 +474,6 @@ def forward_sparse_mla_kvcache_bf16( 2. Load full kv cache from paged memory (with k rope applied) 3. Call FlashMLA sparse attention kernel for sparse prefill/decode """ - assert isinstance(attn_metadata, DSAtrtllmAttentionMetadata), ( - "DSA requires DSAtrtllmAttentionMetadata" - ) attention_input_type = ( AttentionInputType.generation_only if is_generation else AttentionInputType.context_only ) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/cache_manager.py index 197998c14b08..84bcf2dd1bda 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/cache_manager.py @@ -1327,7 +1327,7 @@ class MiniMaxM3DraftKVCacheView: dtype, pool pointers and block tables from the metadata's manager; the shared manager's are NVFP4, with ``[data, scale]`` pool pointers. This view flows wherever a separate draft manager would (``get_draft_kv_cache_manager`` - and the attention-metadata draft swap), with dtype FP8 and one pool rooted at + and the draft metadata built over it), with dtype FP8 and one pool rooted at the draft layer's K page: slot ``s`` is K page ``s * scale`` and V page ``s * scale + 1`` (see :meth:`MiniMaxM3KVCacheManagerV2.get_kv_subpage_pool`). Everything else, block lifecycle included, stays with the shared manager. diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/kernels/msa_utils.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/kernels/msa_utils.py index 774c21d88126..6cf6a9c589c2 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/kernels/msa_utils.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/kernels/msa_utils.py @@ -184,7 +184,7 @@ def write_msa_phase_kv( write_msa_main_kv( metadata.kv_cache_manager, attn.layer_idx, - metadata.msa_out_cache_loc[token_offset : token_offset + num_tokens], + metadata.msa_write_slots()[token_offset : token_offset + num_tokens], k, v, # k and v are this phase's own token slice, so every row owns a slot. diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py index 34fdeae7fc00..083a34762c2e 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py @@ -10,34 +10,35 @@ * The indexer calls fmha_sm100 directly to produce the per-query selected block indices, which the model layer threads through forward_args.sparse_backend_args. - * MiniMaxM3MsaSparseAttentionMetadata subclasses TrtllmAttentionMetadata and - stores its per-forward MSA tensors in CUDA-graph-stable buffers. - The buffers are allocated once in __post_init__ via - get_empty(capture_graph=...), and prepare() copies the per-step values - into them. The standard CUDAGraphRunner clones one metadata per graph - batch size (create_cuda_graph_metadata), so no per-batch-size cache is - needed here. + * MiniMaxM3MsaMetadata extends TrtllmAttentionMetadata and stores its + per-forward MSA tensors in CUDA-graph-stable buffers. The buffers are + allocated once per metadata in allocate() via get_empty(capture_graph=...), + and prepare() copies the per-step values into them. The standard + CUDAGraphRunner clones one metadata per graph batch size + (create_cuda_graph_metadata), so no per-batch-size cache is needed here. * With Eagle3 a decode row has 1 + draft_len query tokens. Slots and valid-block counts are per token, and the decode kernels take that uniform query length from msa_decode_span. on_update_kv_lens re-derives the - per-request lengths, the slots and the counts after the overlap scheduler - corrects kv_lens on device. + per-request lengths, the slots and the counts in prepare() when the + overlap scheduler's correction makes the device kv_lens differ from the + host values. -The classes subclass TrtllmAttention and TrtllmAttentionMetadata, imported at +The classes build on TrtllmAttention and TrtllmAttentionMetadata, imported at module scope. That is cycle-free only because the dependency runs one way: the -two FMHA libraries reach the kernels through ...minimax_m3.kernels and never -load this module, so trtllm's import chain does not come back here. +two FMHA libraries reach the kernels through ...minimax_m3.kernels and load +this module only lazily, at call time, to look up MiniMaxM3MsaMetadata, so +trtllm's import chain does not come back here. """ from __future__ import annotations -from dataclasses import dataclass from typing import NamedTuple, Optional, Tuple import numpy as np import torch from tensorrt_llm._torch.attention.backends.interface import AttentionForwardArgs +from tensorrt_llm._torch.attention.backends.sparse.params import SparseAlgorithmExtension from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor import h2d_cache, step_inputs from tensorrt_llm._torch.pyexecutor.h2d_cache import copy_h2d_if_changed, copy_h2d_pool_shared @@ -122,8 +123,8 @@ def _host_prefix_sum(lens_cpu: torch.Tensor, dtype: torch.dtype) -> torch.Tensor # (the device and every captured graph only read it), so the shadow describes the device rows; False always uploads it. SKIP_UNCHANGED_BLOCK_TABLE = True # TAIL_FILL_IN_FORWARD: prepare() records where msa_out_cache_loc's live slots end and the forward invalidates the rest -# (on_update_kv_lens, which opens every forward, so a captured forward fills its own tail at each replay) instead of an -# eager fill between the decode graphs; False fills it in prepare() as before. +# (msa_write_slots, which opens the first cache write in every forward, so a captured forward fills its own tail at each +# replay) instead of an eager fill between the decode graphs; False fills it in prepare() as before. TAIL_FILL_IN_FORWARD = True @@ -145,9 +146,8 @@ class MsaDecodeSpan(NamedTuple): query_len: int -@dataclass(init=False) -class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): - """TrtllmAttentionMetadata for MiniMax-M3 MSA sparse layers. +class MiniMaxM3MsaMetadata(SparseAlgorithmExtension): + """TrtllmAttentionMetadata extension for MiniMax-M3 MSA sparse layers. A step is prepared for a fixed division of labour: its context rows run on fmha_sm100 through MsaPrefillFmha, and its generation rows on MiniMax-M3's @@ -172,13 +172,86 @@ class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): With Eagle3 a decode row has 1 + draft_len query tokens: the span's query_len is that count, slots and valid-block counts are per token, and - the overlap scheduler corrects kv_lens on device after prepare(). The + the overlap scheduler's correction makes the device kv_lens differ from + the staged host values. The decode kernels read their lengths from msa_seq_lens_cuda, so on_update_kv_lens patches that buffer, the slots and the counts. The fmha_sm100 plans cover context rows only, whose lengths the correction never touches, so they need no patch. """ + fields = frozenset( + { + "_msa_buffers_ready", + "_msa_context_prefix_bounds", + "_msa_fields_ready", + "_msa_kv_lens_cpu", + "_msa_kv_lens_dynamic", + "_msa_live_batch", + "_msa_max_kv_len_all", + "_msa_max_q_len", + "_msa_params", + "_msa_prefill_n_valid_blocks", + "_msa_qo_lens_cpu", + "_msa_qo_offset_cpu", + "_msa_subpages_per_slot", + "_msa_total_k", + "_msa_total_k_rows", + "_msa_worst_case_max_k_tiles", + "msa_block_table", + "msa_cu_kv_lens", + "msa_cu_q_lens", + "msa_decode_query_len", + "msa_decode_span", + "msa_kv_indices", + "msa_kv_lens_cpu", + "msa_kv_lens_staged", + "msa_layer_cache_tensors", + "msa_max_kv_len", + "msa_max_score", + "msa_n_valid_blocks", + "msa_out_cache_loc", + "msa_policy", + "msa_prefill_dense_plan", + "msa_prefill_gqa_plan", + "msa_identity_csr", + "msa_prefill_identity", + "msa_prefill_n_valid_blocks", + "msa_prefill_proxy_kv_indices", + "msa_prefill_proxy_plan", + "msa_q_batch_row", + "msa_q_intra", + "msa_qo_lens_cpu", + "msa_qo_lens_cuda", + "msa_qo_offset_cpu", + "msa_seq_lens_cuda", + "msa_short_table", + "msa_short_table_staged", + "msa_subpage_block_table", + "msa_worst_case_max_k_tiles", + } + ) + methods = frozenset( + { + "_build_msa_fields", + "_build_step_plans", + "_create_msa_buffers", + "_ensure_msa_decode_scratch_buffers", + "_msa_attn_plan_rows", + "_msa_kv_lens_may_change", + "_msa_proxy_plan_rows", + "_msa_runs_no_fmha", + "_set_decode_span", + "_stage_host_lengths", + "cache_identity_csr", + "msa_idx_k_cache", + "msa_proxy_max_score_view", + "msa_subpage_rows", + "msa_write_idx_k", + "msa_write_slots", + } + ) + # Graph-stable buffers; consumers slice to the live count at the call # site. Filled once the current step's cache write is prepared. msa_out_cache_loc: Optional[torch.Tensor] = None @@ -190,8 +263,8 @@ class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): msa_kv_indices: Optional[torch.Tensor] = None msa_max_score: Optional[torch.Tensor] = None msa_n_valid_blocks: Optional[torch.Tensor] = None - # Decode-step preparation the model opted into (RuntimeHooks.msa). - msa_policy: MsaRuntimePolicy = MsaRuntimePolicy() + # Decode-step preparation the model opted into (RuntimeHooks.msa), given at creation. + init_fields = {"msa_policy": MsaRuntimePolicy()} # Short-sequence steps (MsaRuntimePolicy.short_sequence_identity): every query token has at most topk valid # blocks, so the selection is the identity table [kv_heads, tokens, topk] = (i < n_valid ? i : -1), staged here # by prepare(). _msa_short_ok says whether this step staged it. @@ -215,8 +288,10 @@ class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): # SUBPAGE_IN_GRAPH: the rows prepare() staged for the forward to expand (0: none), and whether this forward did. _msa_subpage_rows: int = 0 _msa_subpage_expanded: bool = False - # TAIL_FILL_IN_FORWARD: the first msa_out_cache_loc row past this step's live slots (None: prepare() filled it). + # TAIL_FILL_IN_FORWARD: the first msa_out_cache_loc row past this step's live slots (None: prepare() filled it), + # and whether this forward has already filled it. _msa_tail_start: Optional[int] = None + _msa_tail_filled: bool = False # Inputs for on_update_kv_lens: each query token's request row and offset # within the request, and the kv_lens prepare() staged (the upper bound the # correction is clamped to). The write slots are re-derived from @@ -267,8 +342,8 @@ class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): # staging are skipped and non-speculative steps run as before. _msa_kv_lens_dynamic: bool = False - def __post_init__(self) -> None: - super().__post_init__() + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) params = self.sparse_metadata_params self._msa_params = params if isinstance(params, MiniMaxM3SparseMetadataParams) else None self._create_msa_buffers() @@ -297,18 +372,19 @@ def _stage_host_lengths(self) -> None: what lets the planners stage them with non-blocking copies instead of copying out of pageable memory, so it is worth paying for once a step. """ + md = self.metadata def as_pinned_int32(lens: torch.Tensor) -> torch.Tensor: - rows = lens[: self.num_seqs] + rows = lens[: md.num_seqs] return maybe_pin_memory(rows.to(torch.int32) if rows.dtype != torch.int32 else rows) - seq_lens = self.seq_lens - kv_lens = getattr(self, "kv_lens", None) + seq_lens = md.seq_lens + kv_lens = getattr(md, "kv_lens", None) # The host kv_lens carries num_extra_kv_tokens (speculative draft # slots); MSA needs the attended length, which is what kv_lens_cuda # holds. Every consumer below, msa_seq_lens_cuda included, inherits # the correction from here. - params = self.kv_cache_params + params = md.kv_cache_params extra = int(params.num_extra_kv_tokens) if params is not None else 0 if kv_lens is not None and extra: kv_lens = kv_lens - extra @@ -383,6 +459,30 @@ def msa_worst_case_max_k_tiles(self) -> int: """ return self._msa_worst_case_max_k_tiles + @property + def msa_prefill_identity(self) -> Optional[torch.Tensor]: + """This step's identity selection [total_q, topk] of a short pure-prefill step, or None.""" + return self._msa_prefill_identity + + @property + def msa_identity_csr(self) -> Optional[tuple]: + """(identity table, key, k2q CSR worklist) a layer of this step built for its identity selection. + + The worklist is layer-invariant for an identity selection, so the + first layer of a step builds it and the later layers reuse it while + the table is this step's (``msa_prefill_identity``) and the key matches. + """ + return self._msa_identity_csr + + def cache_identity_csr(self, entry: tuple) -> None: + """Keep ``entry`` for the later layers of this step; see ``msa_identity_csr``.""" + self._msa_identity_csr = entry + + @property + def msa_short_table_staged(self) -> bool: + """Whether this step staged the identity selection in ``msa_short_table``.""" + return self._msa_short_ok + @property def msa_prefill_n_valid_blocks(self) -> Optional[torch.Tensor]: """Device int32 valid-block count for the fmha_sm100 proxy rows, or None @@ -400,9 +500,8 @@ def msa_subpage_rows(self, row_first: int, row_last: int) -> Tuple[Optional[torc return None, 0 rows = self._msa_subpage_rows if rows > 0 and not self._msa_subpage_expanded: - # Once per forward (on_update_kv_lens, which opens every forward, resets it), from the block table the - # pack landed, over the rows prepare() staged; a captured forward records it once, so every replay - # expands its own rows. + # Once per forward (prepare and a draft refresh reset it), from the block table the pack landed, over + # the rows prepare() staged; a captured forward records it once, so every replay expands its own rows. write_subpage_block_table( self.msa_block_table[:rows], self._msa_subpages_per_slot, table[:rows] ) @@ -416,7 +515,8 @@ def _msa_main_kv_is_fp8(self) -> bool: sparse-prefill path selects the FP8 AOT kernels. Mirrors the k_paged dtype check in run_msa_prefill_gqa. """ - kv_cache_manager = self.kv_cache_manager + md = self.metadata + kv_cache_manager = md.kv_cache_manager return kv_cache_manager is not None and kv_cache_manager.dtype in ( DataType.FP8, DataType.NVFP4, @@ -432,8 +532,9 @@ def _validate_decode_kernel_support(self) -> None: not the layer, fixes the page size, the index dtype and the sub-page pool. """ + md = self.metadata params = self._msa_params - kv_cache_manager = self.kv_cache_manager + kv_cache_manager = md.kv_cache_manager if params is None or kv_cache_manager is None: # No MSA geometry to check, as for a structural test's metadata. return @@ -443,7 +544,7 @@ def _validate_decode_kernel_support(self) -> None: # num_index_heads * query tokens into one Q block, which bounds the # draft length it can verify. decode_query_len = self._msa_max_decode_query_len() - num_index_heads = params.sharded_index_head_count(self.mapping) + num_index_heads = params.sharded_index_head_count(md.mapping) if not self._cutedsl_indexer_supported( num_index_heads=num_index_heads, page_size=page_size, @@ -470,7 +571,8 @@ def _create_msa_buffers(self) -> None: max_num_tokens for cache slots, max_num_sequences * max_blocks_per_seq for the page table, and worst-case max_k_tiles for proxy scratch. """ - kv_cache_manager = self.kv_cache_manager + md = self.metadata + kv_cache_manager = md.kv_cache_manager self._msa_buffers_ready = False if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): return @@ -487,35 +589,35 @@ def _create_msa_buffers(self) -> None: for layer_idx in getattr(kv_cache_manager, "sparse_layer_ids", ()) if layer_idx in kv_cache_manager.layer_offsets } - capture_graph = self.is_cuda_graph - buffers = self.cuda_graph_buffers - max_num_sequences = int(self.max_num_sequences) + capture_graph = md.is_cuda_graph + buffers = md.cuda_graph_buffers + max_num_sequences = int(md.max_num_sequences) max_blocks_per_seq = int(kv_cache_manager.max_blocks_per_seq) max_total_pages = max_num_sequences * max_blocks_per_seq - max_num_tokens = int(self.max_num_tokens) + max_num_tokens = int(md.max_num_tokens) - self.msa_out_cache_loc = self.get_empty( + self.msa_out_cache_loc = md.get_empty( buffers, (max_num_tokens,), cache_name="msa_out_cache_loc", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_kv_indices = self.get_empty( + self.msa_kv_indices = md.get_empty( buffers, (max_total_pages,), cache_name="msa_kv_indices", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_block_table = self.get_empty( + self.msa_block_table = md.get_empty( buffers, (max_num_sequences, max_blocks_per_seq), cache_name="msa_block_table", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_seq_lens_cuda = self.get_empty( + self.msa_seq_lens_cuda = md.get_empty( buffers, (max_num_sequences,), cache_name="msa_seq_lens_cuda", @@ -523,21 +625,21 @@ def _create_msa_buffers(self) -> None: capture_graph=capture_graph, ) if getattr(kv_cache_manager, "dtype", None) == DataType.NVFP4: - self.msa_qo_lens_cuda = self.get_empty( + self.msa_qo_lens_cuda = md.get_empty( buffers, (max_num_sequences,), cache_name="msa_qo_lens_cuda", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_cu_q_lens = self.get_empty( + self.msa_cu_q_lens = md.get_empty( buffers, (max_num_sequences + 1,), cache_name="msa_cu_q_lens", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_cu_kv_lens = self.get_empty( + self.msa_cu_kv_lens = md.get_empty( buffers, (max_num_sequences + 1,), cache_name="msa_cu_kv_lens", @@ -548,7 +650,7 @@ def _create_msa_buffers(self) -> None: # pool's layout for the life of the manager. self._msa_subpages_per_slot = uniform_subpages_per_slot(kv_cache_manager) if self._msa_subpages_per_slot > 0: - self.msa_subpage_block_table = self.get_empty( + self.msa_subpage_block_table = md.get_empty( buffers, (max_num_sequences, 2, max_blocks_per_seq), cache_name="msa_subpage_block_table", @@ -556,21 +658,21 @@ def _create_msa_buffers(self) -> None: capture_graph=capture_graph, ) # Inputs for on_update_kv_lens. - self.msa_q_batch_row = self.get_empty( + self.msa_q_batch_row = md.get_empty( buffers, (max_num_tokens,), cache_name="msa_q_batch_row", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_q_intra = self.get_empty( + self.msa_q_intra = md.get_empty( buffers, (max_num_tokens,), cache_name="msa_q_intra", dtype=torch.int32, capture_graph=capture_graph, ) - self.msa_kv_lens_staged = self.get_empty( + self.msa_kv_lens_staged = md.get_empty( buffers, (max_num_sequences,), cache_name="msa_kv_lens_staged", @@ -586,13 +688,13 @@ def _create_msa_buffers(self) -> None: fmha_sm100 = require_msa_module() max_k_tiles = _worst_case_proxy_max_k_tiles( fmha_sm100, - num_index_heads=params.sharded_index_head_count(self.mapping), + num_index_heads=params.sharded_index_head_count(md.mapping), kv_cache_manager=kv_cache_manager, max_batch=max_num_sequences, ) self._msa_worst_case_max_k_tiles = int(max_k_tiles) self._alloc_msa_proxy_scratch( - num_index_heads=params.sharded_index_head_count(self.mapping), + num_index_heads=params.sharded_index_head_count(md.mapping), max_tokens=self._msa_max_decode_tokens(), max_k_tiles=max_k_tiles, capture_graph=capture_graph, @@ -607,7 +709,8 @@ def _msa_max_decode_query_len(self) -> int: one source for both the up-front kernel validation and the scratch sizing. A run without speculative decoding reports 1. """ - draft_len = int(getattr(self.kv_cache_manager, "max_total_draft_tokens", 0) or 0) + md = self.metadata + draft_len = int(getattr(md.kv_cache_manager, "max_total_draft_tokens", 0) or 0) return 1 + max(0, draft_len) def _msa_max_decode_tokens(self) -> int: @@ -618,9 +721,10 @@ def _msa_max_decode_tokens(self) -> int: 16384, which also keeps a whole-batch fmha_sm100 proxy plan under its 65536 total_q * num_qo_heads limit with 4 index heads. """ - max_seqs = int(self.max_num_sequences) + md = self.metadata + max_seqs = int(md.max_num_sequences) tokens = max_seqs * self._msa_max_decode_query_len() - max_toks = int(self.max_num_tokens or 0) + max_toks = int(md.max_num_tokens or 0) if max_toks > 0: tokens = min(tokens, max_toks) return max(max_seqs, min(tokens, 16384)) @@ -639,15 +743,16 @@ def _alloc_msa_proxy_scratch( the batch size under speculative verify), so one allocation serves every decode step. msa_proxy_max_score_view slices the per-step shape. """ - buffers = self.cuda_graph_buffers - self.msa_max_score = self.get_empty( + md = self.metadata + buffers = md.cuda_graph_buffers + self.msa_max_score = md.get_empty( buffers, (num_index_heads * max_k_tiles * max_tokens,), cache_name="msa_max_score", dtype=torch.float32, capture_graph=capture_graph, ) - self.msa_n_valid_blocks = self.get_empty( + self.msa_n_valid_blocks = md.get_empty( buffers, (max_tokens,), cache_name="msa_n_valid_blocks", @@ -664,6 +769,7 @@ def _ensure_msa_decode_scratch_buffers( required_max_k_tiles: int, ) -> None: """Ensure proxy scratch buffers exist and cover the current plan.""" + md = self.metadata max_tokens = max(int(max_batch), self._msa_max_decode_tokens()) required_numel = num_index_heads * required_max_k_tiles * max_tokens if self.msa_max_score is not None: @@ -676,7 +782,7 @@ def _ensure_msa_decode_scratch_buffers( ) return - kv_cache_manager = self.kv_cache_manager + kv_cache_manager = md.kv_cache_manager if kv_cache_manager is None: return @@ -709,36 +815,37 @@ def _ensure_prefill_n_valid_buffer(self, total_q: int, device: torch.device) -> """ buf = self._msa_prefill_n_valid_buf if buf is None or buf.numel() < total_q or buf.device != device: - cap = max(int(total_q), int(getattr(self, "max_num_tokens", 0) or 0), 1) + cap = max(int(total_q), int(getattr(self.metadata, "max_num_tokens", 0) or 0), 1) buf = torch.empty(cap, dtype=torch.int32, device=device) self._msa_prefill_n_valid_buf = buf return buf def _stage_msa_short_table(self, n_valid: torch.Tensor, params) -> None: """Stage the identity block table for short-sequence steps (see msa_short_table).""" + md = self.metadata self._msa_short_ok = False if ( not self.msa_policy.short_sequence_identity or n_valid.numel() == 0 - or self.num_contexts > 0 + or md.num_contexts > 0 ): return topk = int(MSA_REQUIRED_TOPK) if int(n_valid.max()) > topk: return - _, num_kv_heads = params.sharded_head_counts(self.mapping) + _, num_kv_heads = params.sharded_head_counts(md.mapping) total_q = int(n_valid.shape[0]) - max_tokens = max(int(self.max_num_sequences), self._msa_max_decode_tokens(), total_q) + max_tokens = max(int(md.max_num_sequences), self._msa_max_decode_tokens(), total_q) if ( self.msa_short_table is None or self.msa_short_table.numel() < num_kv_heads * max_tokens * topk ): - self.msa_short_table = self.get_empty( - self.cuda_graph_buffers, + self.msa_short_table = md.get_empty( + md.cuda_graph_buffers, (num_kv_heads * max_tokens * topk,), cache_name="msa_short_table", dtype=torch.int32, - capture_graph=self.is_cuda_graph, + capture_graph=md.is_cuda_graph, ) ar = torch.arange(topk, dtype=torch.int32) table = torch.where(ar[None, :] < n_valid.to(torch.int32)[:, None], ar[None, :], -1) @@ -749,7 +856,22 @@ def _stage_msa_short_table(self, n_valid: torch.Tensor, params) -> None: self._msa_short_ok = True def prepare(self) -> None: - super().prepare() + self._prepare_step() + if self.metadata.kv_lens_offsets is not None: + # The core corrected the device KV lengths after the host staged + # the full-acceptance values. + self.on_update_kv_lens() + + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + # A new forward: the first dense layer that reads the sub-page table + # expands it again (msa_subpage_rows), and the first cache writer + # refills the unwritten tail (msa_write_slots). + self._msa_subpage_expanded = False + self._msa_tail_filled = False + self.on_update_kv_lens() + + def _prepare_step(self) -> None: self._check_beam_width() # Everything below reads these. self._stage_host_lengths() @@ -767,10 +889,11 @@ def _check_beam_width(self) -> None: beam_width rows, so the block table and lengths handed to them would cover only the first 1 / beam_width of the batch. """ - if self.beam_width != 1: + md = self.metadata + if md.beam_width != 1: raise NotImplementedError( "MiniMax-M3 MSA attention does not support beam search, but this " - f"step has beam_width={self.beam_width}. Use beam_width=1 or the " + f"step has beam_width={md.beam_width}. Use beam_width=1 or the " "non-MSA MiniMax-M3 backend." ) @@ -791,13 +914,14 @@ def _set_decode_span(self) -> None: and may not exceed what _validate_decode_kernel_support settled the scorer for. """ + md = self.metadata self._msa_decode_span = None self._msa_max_kv_len = 0 qo_lens_cpu = self.msa_qo_lens_cpu kv_lens_cpu = self.msa_kv_lens_cpu if qo_lens_cpu is None or kv_lens_cpu is None: return - row_first = int(self.num_contexts or 0) + row_first = int(md.num_contexts or 0) row_last = int(qo_lens_cpu.shape[0]) if row_first >= row_last: # Pure prefill: no generation row to describe. @@ -833,10 +957,11 @@ def _check_capture_is_pure_decode(self) -> None: fmha_sm100 against the addresses of a step that has passed. Decode needs no such plan, which is why capture is confined to it. """ - if self.is_cuda_graph and int(self.num_contexts or 0) > 0: + md = self.metadata + if md.is_cuda_graph and int(md.num_contexts or 0) > 0: raise RuntimeError( "MiniMax-M3 MSA attention captured a CUDA graph for a step with " - f"{int(self.num_contexts)} context requests. Only pure-decode steps " + f"{int(md.num_contexts)} context requests. Only pure-decode steps " "are graph-safe here; see _build_step_plans." ) @@ -862,9 +987,10 @@ def _msa_proxy_plan_rows(self) -> Optional[Tuple[int, int]]: * mixed: the context prefix only; * pure decode: None, no rows left for the proxy. """ + md = self.metadata span = self._msa_decode_span if span is None: - return (0, int(self.num_seqs)) + return (0, int(md.num_seqs)) return (0, span.row_first) if span.row_first > 0 else None def _msa_attn_plan_rows(self) -> Optional[Tuple[int, int]]: @@ -873,7 +999,8 @@ def _msa_attn_plan_rows(self) -> Optional[Tuple[int, int]]: The context prefix, which is the whole of what fmha_sm100 attends: the generation rows are the decode kernels' and are never planned for it. """ - num_contexts = int(self.num_contexts or 0) + md = self.metadata + num_contexts = int(md.num_contexts or 0) return (0, num_contexts) if num_contexts > 0 else None def _msa_index_kv_dtype(self) -> torch.dtype: @@ -882,7 +1009,8 @@ def _msa_index_kv_dtype(self) -> torch.dtype: The CuTe DSL scorer requires index Q and K to match, and run_indexer casts Q to the cache dtype, so the cache decides what the scorer sees. """ - indexer_kv_dtype = str(getattr(self.kv_cache_manager, "indexer_kv_dtype", "bf16")) + md = self.metadata + indexer_kv_dtype = str(getattr(md.kv_cache_manager, "indexer_kv_dtype", "bf16")) return torch.float8_e4m3fn if indexer_kv_dtype == "fp8" else torch.bfloat16 def _cutedsl_indexer_supported( @@ -909,12 +1037,13 @@ def _msa_kv_lens_may_change(self) -> bool: max_total_draft_tokens is set by the engine's update_spec_dec_param; the other checks mirror TrtllmAttentionMetadata's spec_active. """ - params = self.kv_cache_params - runtime_features = self.runtime_features + md = self.metadata + params = md.kv_cache_params + runtime_features = md.runtime_features return bool( - self.max_total_draft_tokens - or self.draft_kv_cache_manager is not None - or self.is_spec_decoding_enabled + md.max_total_draft_tokens + or md.draft_kv_cache_manager is not None + or md.is_spec_decoding_enabled or (params is not None and params.num_extra_kv_tokens) or (runtime_features is not None and runtime_features.has_speculative_draft_tokens) ) @@ -922,6 +1051,9 @@ def _msa_kv_lens_may_change(self) -> bool: def _fill_tail_in_forward(self) -> None: """TAIL_FILL_IN_FORWARD: -1 over msa_out_cache_loc past the live slots prepare() staged (see the prepare() comment on padded rows), before anything in the forward reads it.""" + if getattr(self, "_msa_tail_filled", False): + return + self._msa_tail_filled = True start = self._msa_tail_start buf = self.msa_out_cache_loc if start is not None and buf is not None and start < buf.shape[0]: @@ -931,12 +1063,20 @@ def _fill_tail_in_forward(self) -> None: ) buf[start:].fill_(-1) + def msa_write_slots(self, num_tokens: Optional[int] = None) -> Optional[torch.Tensor]: + """Return this forward's write slots after invalidating any deferred tail.""" + self._fill_tail_in_forward() + buf = self.msa_out_cache_loc + if buf is None or num_tokens is None: + return buf + return buf[:num_tokens] + def on_update_kv_lens(self) -> None: """Re-derive lengths, slots and valid-block counts from the corrected kv_lens_cuda. - The overlap scheduler shortens kv_lens on device after prepare() staged - full-acceptance values (CUDA-graph warmup restores them the same way - between forwards). Shrinking keeps the staged page table valid, so only + The overlap scheduler's correction (``kv_lens_offsets``) shortens the + device kv_lens below the full-acceptance values prepare() staged from + the host. Shrinking keeps the staged page table valid, so only the per-row lengths and what derives from them are patched; the clamp to msa_kv_lens_staged enforces that. Device-only, capture-safe and idempotent; skipped without speculative decoding. @@ -950,17 +1090,14 @@ def on_update_kv_lens(self) -> None: extend_ctx promotes speculative generation rows into the context prefix. Staged host bounds remain valid upper bounds as lengths shrink. """ - # A new forward: the first dense layer that reads the sub-page table expands it again (msa_subpage_rows). - self._msa_subpage_expanded = False - self._fill_tail_in_forward() - super().on_update_kv_lens() if not self._msa_fields_ready or not self._msa_kv_lens_dynamic: return - batch = int(self.num_seqs) - total_q = int(self.num_tokens) + md = self.metadata + batch = int(md.num_seqs) + total_q = int(md.num_tokens) if batch <= 0 or total_q <= 0: return - page = int(self.kv_cache_manager.tokens_per_block) + page = int(md.kv_cache_manager.tokens_per_block) n_valid_buf = ( self._msa_prefill_n_valid_blocks if self._msa_prefill_n_valid_blocks is not None @@ -968,9 +1105,9 @@ def on_update_kv_lens(self) -> None: ) kv_update = self.msa_policy.kv_update if kv_update is not None and kv_update( - self.kv_lens_cuda, + md.kv_lens_cuda, self.msa_kv_lens_staged, - self.seq_lens_cuda, + md.seq_lens_cuda, self.msa_q_batch_row, self.msa_q_intra, self.msa_block_table, @@ -985,14 +1122,14 @@ def on_update_kv_lens(self) -> None: return # kv_lens_cuda is the attended length (no num_extra_kv_tokens), the same # domain as the staged bound and msa_seq_lens_cuda. - kv_true = torch.minimum(self.kv_lens_cuda[:batch], self.msa_kv_lens_staged[:batch]) + kv_true = torch.minimum(md.kv_lens_cuda[:batch], self.msa_kv_lens_staged[:batch]) self.msa_seq_lens_cuda[:batch].copy_(kv_true) if self.msa_cu_kv_lens is not None: self.msa_cu_kv_lens[0].zero_() torch.cumsum(kv_true, 0, out=self.msa_cu_kv_lens[1 : batch + 1]) qbr = self.msa_q_batch_row[:total_q].to(torch.long) - qo_dev = self.seq_lens_cuda[:batch] + qo_dev = md.seq_lens_cuda[:batch] # Position each query token attends up to. pos = kv_true[qbr] - qo_dev[qbr] + self.msa_q_intra[:total_q] @@ -1001,7 +1138,7 @@ def on_update_kv_lens(self) -> None: # pos % page. Positions only move down, so the page is one prepare() # staged for that request; a padding row (pos -1) keeps its own first # slot, as on the host. - page = int(self.kv_cache_manager.tokens_per_block) + page = int(md.kv_cache_manager.tokens_per_block) table = self.msa_block_table pos_slot = pos.clamp_min(0) block_col = torch.div(pos_slot, page, rounding_mode="floor").clamp( @@ -1039,6 +1176,7 @@ def _build_step_plans(self) -> None: (_check_capture_is_pure_decode), and so lets them be plain tuples of per-step tensors. """ + md = self.metadata self._msa_prefill_proxy_plan = None self._msa_prefill_proxy_kv_indices = None self._msa_prefill_gqa_plan = None @@ -1052,21 +1190,21 @@ def _build_step_plans(self) -> None: params = self._msa_params if params is None: return - num_index_heads = params.sharded_index_head_count(self.mapping) + num_index_heads = params.sharded_index_head_count(md.mapping) qo_lens_cpu = self.msa_qo_lens_cpu kv_lens_cpu = self.msa_kv_lens_cpu qo_offset_cpu = self.msa_qo_offset_cpu if qo_lens_cpu is None or kv_lens_cpu is None or qo_offset_cpu is None: return batch = int(qo_lens_cpu.shape[0]) - page_size = int(self.kv_cache_manager.tokens_per_block) + page_size = int(md.kv_cache_manager.tokens_per_block) if self._msa_runs_no_fmha(): # Pure decode: no plan to build, but the scorer still writes its # scores into the proxy scratch and reads the valid-block count. self._ensure_msa_decode_scratch_buffers( num_index_heads=num_index_heads, - max_batch=int(self.max_num_sequences), - capture_graph=self.is_cuda_graph, + max_batch=int(md.max_num_sequences), + capture_graph=md.is_cuda_graph, # No proxy plan, so the worst case is the only bound available. required_max_k_tiles=self._msa_worst_case_max_k_tiles, ) @@ -1082,7 +1220,7 @@ def _build_step_plans(self) -> None: return fmha_sm100 = require_msa_module() - num_q_heads, num_kv_heads = params.sharded_head_counts(self.mapping) + num_q_heads, num_kv_heads = params.sharded_head_counts(md.mapping) # The main-attention GQA and dense plans need use_fp8_kvcache so the # inline sparse-prefill kernel selection matches an FP8 paged cache. # The proxy runs over the bf16 index-K cache, so it never needs the @@ -1145,7 +1283,7 @@ def plan_for(rows: Optional[Tuple[int, int]], **plan_kwargs) -> Optional[tuple]: # Sparse layers: kv_block_num=topk limits attention to top-k blocks. # NVFP4 sparse layers run the MSA CSR kernel, which takes no plan. self._msa_prefill_gqa_plan = plan_for( - None if self.kv_cache_manager.dtype == DataType.NVFP4 else attn_rows, + None if md.kv_cache_manager.dtype == DataType.NVFP4 else attn_rows, num_qo_heads=num_q_heads, num_kv_heads=num_kv_heads, kv_block_num=params.topk, @@ -1161,7 +1299,7 @@ def plan_for(rows: Optional[Tuple[int, int]], **plan_kwargs) -> Optional[tuple]: # Stage the valid-block count to the device once for the whole step # (see msa_prefill_n_valid_blocks). if total_q > 0: - dev_buf = self._ensure_prefill_n_valid_buffer(total_q, _cache_device(self)) + dev_buf = self._ensure_prefill_n_valid_buffer(total_q, _cache_device(md)) dev_buf[:total_q].copy_(n_valid_host.to(torch.int32), non_blocking=True) self._msa_prefill_n_valid_blocks = dev_buf[:total_q] if short_prefill: @@ -1178,25 +1316,28 @@ def _build_msa_fields(self) -> None: buffers. The transient builder tensors are discarded. With speculative decoding the inputs for on_update_kv_lens are staged as well. """ + md = self.metadata self._msa_fields_ready = False - # Set again below once this step's live slots are staged; until then the forward fills nothing. - self._msa_tail_start = None if not self._msa_buffers_ready: return - request_ids = self.request_ids + request_ids = md.request_ids qo_lens_cpu = self.msa_qo_lens_cpu kv_lens_cpu = self.msa_kv_lens_cpu qo_offset_cpu = self.msa_qo_offset_cpu if request_ids is None or qo_lens_cpu is None: + self._msa_tail_start = None + self._msa_tail_filled = False _fill_unused_slots(self.msa_out_cache_loc, 0) return batch_size = int(qo_lens_cpu.shape[0]) if batch_size == 0: + self._msa_tail_start = None + self._msa_tail_filled = False _fill_unused_slots(self.msa_out_cache_loc, 0) return - kv_cache_manager = self.kv_cache_manager - cache_device = _cache_device(self) + kv_cache_manager = md.kv_cache_manager + cache_device = _cache_device(md) page_size = int(kv_cache_manager.tokens_per_block) # Built in prepare() (outside capture), so these transients are @@ -1234,10 +1375,10 @@ def _build_msa_fields(self) -> None: # have to describe the same rows. The mapping emits one slot per new # token, so they agree unless a caller staged lengths this metadata's # seq_lens does not match. - if total_new_tokens != int(self.num_tokens): + if total_new_tokens != int(md.num_tokens): raise ValueError( f"MSA slot mapping covers {total_new_tokens} new tokens, but the " - f"step's token count is {int(self.num_tokens)}." + f"step's token count is {int(md.num_tokens)}." ) if kv_indices is not None and int(kv_indices.shape[0]) > self.msa_kv_indices.shape[0]: raise ValueError( @@ -1258,6 +1399,7 @@ def _build_msa_fields(self) -> None: # would make its shape dynamic, and a negative slot is what makes those # rows cache-write no-ops instead. Invalidate the unwritten tail so they # cannot reuse the previous step's live slots, which address real pages. + self._msa_tail_filled = False if TAIL_FILL_IN_FORWARD: self._msa_tail_start = total_new_tokens else: @@ -1338,7 +1480,7 @@ def _build_msa_fields(self) -> None: # NVFP4 sparse layer's CSR kernel covers once the ported decode kernels # take the generation suffix. Sizing that call from the whole batch # would let a long generation row inflate its worklist. - context_rows = min(int(self.num_contexts or 0), batch_size) + context_rows = min(int(md.num_contexts or 0), batch_size) if context_rows > 0: prefix_kv = kv_lens_cpu[:context_rows].to(torch.int64) self._msa_context_prefix_bounds = ( @@ -1397,6 +1539,9 @@ def _build_msa_fields(self) -> None: maybe_pin_memory(torch.from_numpy(intra_np)), pack=True, ) + self.msa_q_intra[:total_new_tokens].copy_( + maybe_pin_memory(torch.from_numpy(intra_np)), non_blocking=True + ) # The staged lens are the upper bound on_update_kv_lens clamps to: the # same attended lengths msa_seq_lens_cuda was just filled from, kept # apart so the clamp stays fixed however often the hook runs. @@ -1405,20 +1550,22 @@ def _build_msa_fields(self) -> None: def msa_idx_k_cache(self, layer_idx: int) -> torch.Tensor: """Return the paged index-K cache in the HND layout MSA consumes.""" - return self.kv_cache_manager.get_index_k_buffer(layer_idx) + md = self.metadata + return md.kv_cache_manager.get_index_k_buffer(layer_idx) def msa_write_idx_k(self, layer_idx: int, idx_k: torch.Tensor) -> None: """Write the new-token index-K into the side cache at out_cache_loc.""" + md = self.metadata cache = self.msa_idx_k_cache(layer_idx) sparse_index_dim = int(cache.shape[-1]) num_tokens = int(idx_k.shape[0]) write_kv_slots( cache, - self.msa_out_cache_loc[:num_tokens], + self.msa_write_slots(num_tokens), idx_k.reshape(num_tokens, 1, sparse_index_dim), # idx_k arrives over the padded token extent; the live prefix is # where msa_out_cache_loc stops holding real slots. - int(self.num_tokens), + int(md.num_tokens), layout="HND", ) @@ -1452,6 +1599,12 @@ def msa_proxy_max_score_view( return store[:numel].view(num_index_heads, plan_max_k_tiles, num_tokens) +class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the MiniMax-M3 MSA extension attached.""" + + backend_extension_types = (MiniMaxM3MsaMetadata,) + + class MiniMaxM3MsaSparseAttention(TrtllmAttention): """MSA-backed MiniMax-M3 sparse attention.""" @@ -1581,7 +1734,7 @@ def write_layer_caches( scales[:, 0], scales[:, 1], idx_cache, - metadata.msa_out_cache_loc[: k.shape[0]], + metadata.msa_write_slots(int(k.shape[0])), k, v, idx_k, @@ -1595,7 +1748,7 @@ def write_layer_caches( k_view, v_view = buffers[:, 0], buffers[:, 1] idx_cache = metadata.msa_idx_k_cache(layer_idx) if idx_k is not None else None num_tokens = int(k.shape[0]) - out_cache_loc = metadata.msa_out_cache_loc[:num_tokens] + out_cache_loc = metadata.msa_write_slots(num_tokens) if fused_write_layer_caches(k_view, v_view, idx_cache, out_cache_loc, k, v, idx_k): return num_kv_heads = int(k_view.shape[1]) @@ -1693,7 +1846,7 @@ def run_indexer( # proxy plan scores the whole batch instead. gen_first is the span's # first query token: the scorer takes [gen_first, num_tokens) over rows # [ctx_rows, row_last), the proxy the context prefix ahead of both. - identity = metadata._msa_prefill_identity + identity = metadata.msa_prefill_identity if span is None and identity is not None and int(identity.shape[0]) == num_tokens: # Short pure-prefill step: every token selects all its valid blocks (staged once per step). num_kv_heads = int(self.indexer.config.num_kv_heads) @@ -1790,5 +1943,6 @@ def sparse_kv_predict( __all__ = [ "MiniMaxM3MsaSparseAttention", + "MiniMaxM3MsaMetadata", "MiniMaxM3MsaSparseAttentionMetadata", ] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/triton_metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/triton_metadata.py index a1bea4f942fc..39ceb5d17b71 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/triton_metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/triton_metadata.py @@ -21,6 +21,7 @@ import torch from ...trtllm import TrtllmAttentionMetadata +from ..params import SparseAlgorithmExtension from .common import build_paged_kv_slot_mapping @@ -664,34 +665,44 @@ def build_runtime_metadata_from_kv_manager( return meta, out_cache_loc -class MiniMaxM3AttentionMetadata(TrtllmAttentionMetadata): - """:class:`TrtllmAttentionMetadata` that pre-builds MiniMax-M3 metadata. +class MiniMaxM3TritonMetadata(SparseAlgorithmExtension): + """:class:`TrtllmAttentionMetadata` extension that pre-builds MiniMax-M3 metadata. - Subclasses :class:`TrtllmAttentionMetadata` (like - ``DSAtrtllmAttentionMetadata``) so one-model Eagle3 draft layers can run - :class:`TrtllmAttention` on this metadata. + Extends :class:`TrtllmAttentionMetadata` so one-model Eagle3 draft layers + can run :class:`TrtllmAttention` on the same metadata. - Overrides :meth:`prepare` so the M3-sparse + :meth:`prepare` builds the M3-sparse :class:`MiniMaxM3TritonSparseAttentionMetadata` and the per-new-token - ``out_cache_loc`` are built **once per scheduler step**, on the - cache device, before the model forward runs. The result is - stored as ``self.minimax_m3 = {"metadata": m3_meta, - "out_cache_loc": out_cache_loc}`` so the model layer's - ``_dense_forward`` and ``_sparse_forward`` can read it without - any device migration. - - Test paths that build their own metadata can short-circuit by - attaching ``attn_metadata.minimax_m3`` directly before calling - the forward; those paths do not go through :meth:`prepare`. + ``out_cache_loc`` **once per scheduler step**, on the cache device, + before the model forward runs, as ``sparse_metadata`` and + ``out_cache_loc``, so the model layer's ``_dense_forward`` and + ``_sparse_forward`` can read them without any device migration. """ - minimax_m3: Optional[dict] = None + fields = frozenset( + { + "out_cache_loc", + "sparse_metadata", + } + ) + methods = frozenset() + + # This step's M3-sparse metadata and per-new-token cache slots; None until + # prepare() builds them. + sparse_metadata: Optional[MiniMaxM3TritonSparseAttentionMetadata] = None + out_cache_loc: Optional[torch.Tensor] = None # Lazily allocated dict of persistent device buffers used to keep # ``MiniMaxM3TritonSparseAttentionMetadata`` tensor addresses stable # across CUDA-graph capture/replay. None until the first # ``prepare()`` call decides to use them (``is_cuda_graph`` / # graph-stable mode). _m3_static_buffers: Optional[dict] = None + # Eager and every CUDA graph copy use one static buffer set. + shared = frozenset({"_m3_static_buffers"}) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self._m3_static_buffers = self.shared_or("_m3_static_buffers", lambda: None) def _maybe_get_m3_static_buffers( self, cache_device: torch.device, kv_cache_manager @@ -703,7 +714,7 @@ def _maybe_get_m3_static_buffers( needed and caches it on ``self._m3_static_buffers``. We allocate the buffers under two conditions: - * ``self.is_cuda_graph`` is True -- the captured graph + * the metadata's ``is_cuda_graph`` is True -- the captured graph requires stable ``data_ptr()`` across replays; OR * the previous prepare() already allocated buffers -- we keep using them so the algorithm sees the same @@ -714,9 +725,8 @@ def _maybe_get_m3_static_buffers( Returns ``None`` when no static buffers should be used (e.g. eager-only test paths that rely on per-call allocations). """ - need_static = ( - bool(getattr(self, "is_cuda_graph", False)) or self._m3_static_buffers is not None - ) + md = self.metadata + need_static = bool(md.is_cuda_graph) or self._m3_static_buffers is not None if not need_static: return None if self._m3_static_buffers is not None: @@ -740,20 +750,23 @@ def _maybe_get_m3_static_buffers( # Caller-provided hints used by the lazy allocator below # when it sizes the persistent buffers on the first real # prepare() call. - "max_num_sequences_hint": int( - getattr(self, "max_num_sequences", None) or self.max_num_requests - ), + "max_num_sequences_hint": int(md.max_num_sequences or md.max_num_requests), "max_num_tokens_hint": int( - getattr(self, "max_num_tokens", None) - or (int(getattr(self, "max_num_sequences", None) or self.max_num_requests)) + md.max_num_tokens or md.max_num_sequences or md.max_num_requests ), } self._m3_static_buffers = placeholder return placeholder def prepare(self) -> None: - super().prepare() - + self._prepare_step() + if self.metadata.kv_lens_offsets is not None: + # The core corrected the device KV lengths after the host staged + # the full-acceptance values. + self.on_update_kv_lens() + + def _prepare_step(self) -> None: + md = self.metadata # Always rebuild the M3 metadata block on each prepare() # call so it reflects the current scheduler step's seq_lens # / request_ids / num_cached_tokens. Production @@ -768,22 +781,23 @@ def prepare(self) -> None: # over ``req_to_token``/``slot_ids`` reads from freed warmup # memory and either produces wrong tokens or fires # ``Indexing.cu:1515`` ``srcIndex < srcSelectDimSize``. - self.minimax_m3 = None + self.sparse_metadata = None + self.out_cache_loc = None # Production path: build the M3 metadata from the standard # AttentionMetadata fields. Requires kv_cache_manager + the # M3 sparse-cache contract. - kv_cache_manager = getattr(self, "kv_cache_manager", None) + kv_cache_manager = md.kv_cache_manager if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): # Not an M3 KV cache manager: nothing to build. The # forward path will raise a clear error if the M3 # backend ends up dispatched without the M3 cache. return - request_ids = getattr(self, "request_ids", None) - seq_lens = self.seq_lens + request_ids = md.request_ids + seq_lens = md.seq_lens if request_ids is None or seq_lens is None: return - num_contexts = int(getattr(self, "num_contexts", 0) or 0) + num_contexts = int(md.num_contexts or 0) batch_size = int(seq_lens.shape[0]) if batch_size == 0: return @@ -797,14 +811,14 @@ def prepare(self) -> None: cache_device = torch.device(f"cuda:{torch.cuda.current_device()}") seq_lens_cpu = ( - getattr(self, "seq_lens_cpu", None) - if hasattr(self, "seq_lens_cpu") + getattr(md, "seq_lens_cpu", None) + if hasattr(md, "seq_lens_cpu") else seq_lens.detach().to("cpu") ) if seq_lens_cpu is None: seq_lens_cpu = seq_lens.detach().to("cpu") - kv_cache_params = getattr(self, "kv_cache_params", None) + kv_cache_params = md.kv_cache_params num_cached_per_seq = ( kv_cache_params.num_cached_tokens_per_seq if kv_cache_params is not None @@ -860,29 +874,25 @@ def prepare(self) -> None: static_buffers=static_buffers, ) - self.minimax_m3 = { - "metadata": m3_meta, - "out_cache_loc": out_cache_loc, - } + self.sparse_metadata = m3_meta + self.out_cache_loc = out_cache_loc def on_update_kv_lens(self) -> None: """Re-derive the M3 attachment from the corrected ``kv_lens_cuda``. - With the overlap scheduler and speculative decoding, prepare() runs - with optimistic lengths and the engine corrects ``kv_lens_cuda`` on - device before calling this (as DSAtrtllmAttentionMetadata does). + With the overlap scheduler and speculative decoding, the host lengths + are optimistic and the core corrects ``kv_lens_cuda`` on device + (``kv_lens_offsets``). Device-only and idempotent. ``seq_lens_cpu`` / ``max_seqlen_k`` keep the optimistic values; they only bound widths the kernels mask by ``seq_lens``. """ - super().on_update_kv_lens() - attachment = self.minimax_m3 - if not attachment: + meta = self.sparse_metadata + if meta is None: return - meta = attachment["metadata"] - out_cache_loc = attachment["out_cache_loc"] + out_cache_loc = self.out_cache_loc batch = int(meta.slot_ids.shape[0]) - kv_lens = self.kv_lens_cuda[:batch] + kv_lens = self.metadata.kv_lens_cuda[:batch] meta.seq_lens[:batch].copy_(kv_lens) if meta.is_prefill: # Only the K-side prefix moves with rejections; the Q-side layout @@ -903,8 +913,15 @@ def on_update_kv_lens(self) -> None: out_cache_loc[:batch].copy_(derive_decode_cache_slots(meta.req_to_token, kv_lens)) +class MiniMaxM3AttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the MiniMax-M3 Triton extension attached.""" + + backend_extension_types = (MiniMaxM3TritonMetadata,) + + __all__ = [ "MiniMaxM3AttentionMetadata", + "MiniMaxM3TritonMetadata", "MiniMaxM3TritonSparseAttentionMetadata", "allocate_minimax_m3_static_buffers", "build_runtime_metadata_from_kv_manager", diff --git a/tensorrt_llm/_torch/attention/backends/sparse/offload.py b/tensorrt_llm/_torch/attention/backends/sparse/offload.py index b86c658f200f..a89d6ce0a055 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/offload.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/offload.py @@ -276,7 +276,7 @@ def get_sparse_offload_metadata_layers(metadata) -> dict[int, SparseOffloadLayer or metadata.max_draft_tokens or manager.max_draft_len or manager.max_total_draft_tokens - or metadata.draft_kv_cache_manager is not None + or metadata.find("draft_kv_cache_manager", None) is not None ): raise NotImplementedError("Sparse offload requires single-beam, non-speculative execution") manager._validate_sparse_history_policy() @@ -442,6 +442,16 @@ def _sparse_arena_base(self, layer_id: int, role: DataRole) -> int: """ return self._ensure_sparse_batch().get_sparse_cache_base_address(layer_id, role) + def prepare_batch_tables(self, request_ids: list[int], num_contexts: int) -> None: + """Publish the batch before attention metadata copies its block offsets. + + The runtime's host tables, which the copy reads, take their snapshot of + the published batch. + """ + if self._has_sparse_offload_layers(): + self.prepare_sparse_batch(request_ids) + super().prepare_batch_tables(request_ids, num_contexts) + def prepare_sparse_batch(self, request_ids: list[int]) -> None: """Publish scheduler or warmup requests, or join an already published batch. diff --git a/tensorrt_llm/_torch/attention/backends/sparse/params.py b/tensorrt_llm/_torch/attention/backends/sparse/params.py index ee697ecf666e..bc046e44f8e9 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/params.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/params.py @@ -16,11 +16,12 @@ import os from dataclasses import dataclass -from typing import Literal, Optional, Protocol, runtime_checkable +from typing import Literal, Optional import torch from ..cpp_schema import cpp_field +from ..extensions import MetadataExtension _INDEXER_MQA_LOGITS_DEFAULT_ELEM_BUDGET = 1 << 31 _INDEXER_MQA_LOGITS_BYTES_PER_ELEMENT = 4 @@ -45,22 +46,122 @@ def get_indexer_mqa_logits_workspace_bytes( return elem_budget * _INDEXER_MQA_LOGITS_BYTES_PER_ELEMENT -@runtime_checkable -class MTPIndexShareMetadata(Protocol): - """Draft-loop state a sparse backend exposes so MTP can share one selection. +# Buffers of MTPIndexShareExtension that the requiring algorithm allocates. +_INDEX_SHARE_BUFFERS = frozenset({"block_table", "req_idx_per_token", "shared_topk_indices"}) - The three calls are used together for one draft loop, so a backend that - implements only some of them cannot serve the reuse at all. + +class MTPIndexShareExtension(MetadataExtension): + """Draft-loop state shared by the sparse algorithms that reuse one MTP selection. + + The speculative driver brackets an MTP draft loop through the setters; the + algorithm's indexer captures its Top-K selection into ``shared_topk_indices`` + once and reuses it while ``indexer_skip_topk`` is set. The algorithm that + requires this extension (DSA or QSA) also keeps the page tables and the + packed-row request map here, so the shared selection and the tables it + indexes are one state. """ + fields = frozenset( + { + "block_table", + "in_mtp_draft_loop", + "indexer_skip_topk", + "mtp_num_accepted", + "req_idx_per_token", + "shared_topk_indices", + "shared_topk_captured_slots", + } + ) + methods = frozenset( + { + "bind_buffers", + "set_in_mtp_draft_loop", + "set_mtp_num_accepted", + "set_skip_topk", + "reset_mtp_index_share", + } + ) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Page table of the algorithm's indexer cache; allocated by the + # requiring algorithm. + self.block_table: Optional[torch.Tensor] = None + # Request row of each packed token; allocated by the requiring algorithm. + self.req_idx_per_token: Optional[torch.Tensor] = None + # Captured selections reused by the draft loop, and the capture slots + # that hold a selection captured in the current loop. + self.shared_topk_indices: Optional[torch.Tensor] = None + self.shared_topk_captured_slots: set[int] = set() + self.in_mtp_draft_loop = False + self.indexer_skip_topk = False + self.mtp_num_accepted: Optional[torch.Tensor] = None + # A metadata copy starts outside any draft loop. + self.shared_topk_captured_slots = set() + self.in_mtp_draft_loop = False + self.indexer_skip_topk = False + self.mtp_num_accepted = None + + adopts_target_state = True + # The draft loop rebuilds the packed-row request map; the captured + # selections are the draft loop's own. + draft_bound = frozenset( + { + "block_table", + "in_mtp_draft_loop", + "indexer_skip_topk", + "mtp_num_accepted", + } + ) + draft_written = frozenset({"req_idx_per_token"}) + draft_private = frozenset({"shared_topk_indices", "shared_topk_captured_slots"}) + + def bind_buffers(self, **buffers: Optional[torch.Tensor]) -> None: + """Bind buffers the requiring algorithm allocated, by field name. + + Accepts ``block_table``, ``req_idx_per_token`` and + ``shared_topk_indices``; None unbinds one. + """ + unknown = set(buffers) - _INDEX_SHARE_BUFFERS + if unknown: + raise TypeError(f"MTPIndexShareExtension has no buffers {sorted(unknown)}") + for name, buffer in buffers.items(): + setattr(self, name, buffer) + def set_in_mtp_draft_loop(self, active: bool) -> None: - """Mark whether a draft loop is running.""" + """Mark whether a draft loop is running. + + Readiness is per capture slot: a layer may reuse only the row it + captured itself in this loop, so every transition clears it. + """ + self.in_mtp_draft_loop = active + self.shared_topk_captured_slots.clear() def set_mtp_num_accepted(self, num_accepted: Optional[torch.Tensor]) -> None: """Supply the per-request accepted counts used to pick the capture row.""" + self.mtp_num_accepted = num_accepted def set_skip_topk(self, skip: bool) -> None: """Ask the indexer to reuse its captured selection instead of scoring.""" + self.indexer_skip_topk = skip + + def reset_mtp_index_share(self) -> None: + """Clear the draft-loop state the indexer reads.""" + self.set_skip_topk(False) + self.set_in_mtp_draft_loop(False) + self.set_mtp_num_accepted(None) + + +class MTPIndexShareUser(MetadataExtension): + """An algorithm whose indexer keeps its tables and draft-loop state in ``MTPIndexShareExtension``.""" + + requires = (MTPIndexShareExtension,) + fields = frozenset({"index_share"}) + + @property + def index_share(self) -> MTPIndexShareExtension: + """The draft-loop state and tables shared with the MTP driver.""" + return self.metadata.ext(MTPIndexShareExtension) def use_self_sampling_gvr( @@ -98,6 +199,15 @@ class SparseMetadataParams: """Base parameters for sparse attention metadata.""" +class SparseAlgorithmExtension(MetadataExtension): + """Base of a sparse algorithm's metadata state; a metadata carries at most one.""" + + fields = frozenset({"sparse_metadata_params"}) + init_fields = {"sparse_metadata_params": None} + #: The algorithm's settings for allocating and updating its state. + sparse_metadata_params: Optional[SparseMetadataParams] = None + + @dataclass(kw_only=True, slots=True) class SparseBackendForwardArgs: """Sparse inputs passed from an attention module to its backend.""" diff --git a/tensorrt_llm/_torch/attention/backends/sparse/qsa/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/qsa/metadata.py index 76320dfc25b3..16bc018104a6 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/qsa/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/qsa/metadata.py @@ -2,11 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 """Attention metadata for QSA sparse attention.""" -from dataclasses import dataclass, field from typing import Optional import torch +from tensorrt_llm._torch.attention.backends.sparse.params import ( + MTPIndexShareUser, + SparseAlgorithmExtension, +) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from .constants import QSA_KEY_ROLE_INDEX @@ -18,149 +21,133 @@ from .params import QSASparseMetadataParams -# Keep inherited dataclass fields while intercepting one QSA-only constructor arg. -@dataclass(init=False) -class QSAAttentionMetadata(TrtllmAttentionMetadata): - """Extend TRTLLM metadata with graph-stable QSA page and row mappings.""" - - sparse_metadata_params: Optional[QSASparseMetadataParams] = None - qsa_needs_speculative_snapshot: bool = field( - init=False, - default=False, - repr=False, +def _index_share_field(name: str) -> property: + """A property reading and writing ``name`` of the ``MTPIndexShareExtension``.""" + + def get(self): + return getattr(self.index_share, name) + + def set(self, value): + setattr(self.index_share, name, value) + + return property(get, set, doc=f"MTPIndexShareExtension.{name}") + + +class QSAMetadata(SparseAlgorithmExtension, MTPIndexShareUser): + """Graph-stable QSA page and row mappings of a TRTLLM metadata.""" + + fields = frozenset( + { + "_attention_pool_id", + "_cu_seq_lens", + "_page_index_scale", + "_token_arange", + "qsa_block_table", + "qsa_has_local_layers", + "qsa_in_mtp_draft_loop", + "qsa_indexer_skip_topk", + "qsa_logical_positions", + "qsa_mtp_num_accepted", + "qsa_needs_speculative_snapshot", + "qsa_req_idx_per_token", + "qsa_sequence_lengths", + "qsa_shared_topk_captured_slots", + "qsa_shared_topk_indices", + "qsa_shared_topk_visible_blocks", + "qsa_topk_indices", + "qsa_topk_row_starts", + "qsa_visible_blocks", + } ) - qsa_has_local_layers: bool = field( - init=False, - default=False, - repr=False, + methods = frozenset( + { + "_refresh_block_table", + "_refresh_speculative_snapshot_state", + "_refresh_token_mapping", + } ) - def __init__(self, *args: object, **kwargs: object) -> None: - # Forwards the parent dataclass's full field set, so the signature - # stays open rather than restating it. - """Allocate the fixed-shape QSA buffers updated before each graph replay.""" - sparse_attention_config = kwargs.pop("sparse_attention_config", None) - if kwargs.get("sparse_metadata_params") is None and sparse_attention_config is not None: - kwargs["sparse_metadata_params"] = sparse_attention_config.to_sparse_metadata_params() - # The generated parent initializer dispatches to this class's - # ``__post_init__``; keep QSA validation there so both constructor forms - # (lowered params or the legacy user config) enforce the same contract. - super().__init__(*args, **kwargs) - - def __post_init__(self) -> None: + # The page table, packed-row request map and draft-loop state live in the + # MTPIndexShareExtension; these read and write them under QSA's names. + qsa_block_table = _index_share_field("block_table") + qsa_req_idx_per_token = _index_share_field("req_idx_per_token") + qsa_shared_topk_indices = _index_share_field("shared_topk_indices") + qsa_shared_topk_captured_slots = _index_share_field("shared_topk_captured_slots") + qsa_in_mtp_draft_loop = _index_share_field("in_mtp_draft_loop") + qsa_indexer_skip_topk = _index_share_field("indexer_skip_topk") + qsa_mtp_num_accepted = _index_share_field("mtp_num_accepted") + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import Qwen4ExpHybridCacheManager - super().__post_init__() - # Draft-loop state; set by the speculative driver, read by the indexer. - self.qsa_in_mtp_draft_loop = False - self.qsa_indexer_skip_topk = False - self.qsa_mtp_num_accepted: Optional[torch.Tensor] = None - self.qsa_shared_topk_indices: Optional[torch.Tensor] = None + md = self.metadata + self.qsa_needs_speculative_snapshot = False + # The shared selection is allocated below when used. + self.index_share.bind_buffers(shared_topk_indices=None) self.qsa_shared_topk_visible_blocks: Optional[torch.Tensor] = None - self.qsa_shared_topk_captured_slots: set[int] = set() - self.qsa_draft_block_table: Optional[torch.Tensor] = None - if not isinstance(self.kv_cache_manager, Qwen4ExpHybridCacheManager): + if not isinstance(md.kv_cache_manager, Qwen4ExpHybridCacheManager): raise TypeError("QSA sparse attention requires Qwen4ExpHybridCacheManager") - if not isinstance(self.sparse_metadata_params, QSASparseMetadataParams): + params = md.sparse_metadata_params + if not isinstance(params, QSASparseMetadataParams): raise TypeError("QSA sparse attention requires QSASparseMetadataParams") - self.qsa_has_local_layers = self.kv_cache_manager.qsa_position_layer_id is not None + self.qsa_has_local_layers = md.kv_cache_manager.qsa_position_layer_id is not None if not self.qsa_has_local_layers: # A PP rank may own only recurrent layers. It still uses the hybrid # manager and parent metadata, but needs no QSA page views. return - capture_graph = self.is_cuda_graph - buffers = self.cuda_graph_buffers # The sparse kernels index raw V2 pool pages, while the parent metadata # stores attention-operator encoded offsets. Keep an unscaled table. - self.qsa_block_table = self.get_empty( - buffers, - ( - self.max_num_sequences, - self.kv_cache_manager.max_blocks_per_seq, - ), - cache_name="qsa_block_table", - dtype=torch.int32, - capture_graph=capture_graph, + self.index_share.bind_buffers( + block_table=md.get_buffer( + (md.max_num_sequences, md.kv_cache_manager.max_blocks_per_seq), + torch.int32, + "qsa_block_table", + ) ) - self._qsa_attention_pool_id, self._qsa_page_index_scale = ( - self.kv_cache_manager.get_qsa_attention_pool_layout() + self._attention_pool_id, self._page_index_scale = ( + md.kv_cache_manager.get_qsa_attention_pool_layout() ) - self._init_qsa_draft_block_table() # Packed-row mapping shared by compression, selection, and sparse GQA. - self.qsa_req_idx_per_token = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_req_idx_per_token", - dtype=torch.int32, - capture_graph=capture_graph, + self.index_share.bind_buffers( + req_idx_per_token=md.get_buffer( + (md.max_num_tokens,), torch.int32, "qsa_req_idx_per_token" + ) ) - self.qsa_logical_positions = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_logical_positions", - dtype=torch.int64, - capture_graph=capture_graph, + self.qsa_logical_positions = md.get_buffer( + (md.max_num_tokens,), torch.int64, "qsa_logical_positions" ) - self.qsa_sequence_lengths = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_sequence_lengths", - dtype=torch.int32, - capture_graph=capture_graph, + self.qsa_sequence_lengths = md.get_buffer( + (md.max_num_tokens,), torch.int32, "qsa_sequence_lengths" ) - self.qsa_visible_blocks = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_visible_blocks", - dtype=torch.int32, - capture_graph=capture_graph, + self.qsa_visible_blocks = md.get_buffer( + (md.max_num_tokens,), torch.int32, "qsa_visible_blocks" ) # Reuse graph-stable row IDs instead of allocating arange each step. - self._qsa_token_arange = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_token_arange", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self._qsa_token_arange.copy_( + self._token_arange = md.get_buffer((md.max_num_tokens,), torch.int64, "qsa_token_arange") + self._token_arange.copy_( torch.arange( - self.max_num_tokens, - device=self._qsa_token_arange.device, + md.max_num_tokens, + device=self._token_arange.device, dtype=torch.int64, ) ) # The leading zero turns request lengths into packed-row boundaries. - self._qsa_cu_seq_lens = self.get_empty( - buffers, - (self.max_num_sequences + 1,), - cache_name="qsa_cu_seq_lens", - dtype=torch.int64, - capture_graph=capture_graph, + self._cu_seq_lens = md.get_buffer( + (md.max_num_sequences + 1,), torch.int64, "qsa_cu_seq_lens" ) - self._qsa_cu_seq_lens.zero_() + self._cu_seq_lens.zero_() # TopK writes in place; caller ownership keeps the address graph-stable. - self.qsa_topk_indices = self.get_empty( - buffers, - ( - self.max_num_tokens, - self.sparse_metadata_params.block_topk, - ), - cache_name="qsa_topk_indices", - dtype=torch.int32, - capture_graph=capture_graph, + self.qsa_topk_indices = md.get_buffer( + (md.max_num_tokens, params.block_topk), torch.int32, "qsa_topk_indices" ) # Per-row Top-K lower bounds; QSA searches every row from zero. - self.qsa_topk_row_starts = self.get_empty( - buffers, - (self.max_num_tokens,), - cache_name="qsa_topk_row_starts", - dtype=torch.int32, - capture_graph=capture_graph, + self.qsa_topk_row_starts = md.get_buffer( + (md.max_num_tokens,), torch.int32, "qsa_topk_row_starts" ) self.qsa_topk_row_starts.zero_() @@ -170,101 +157,49 @@ def __post_init__(self) -> None: # selections differ. ``shared_topk_visible_blocks`` records how many # compressed groups were complete when the row was captured, so the # reuse can tell which groups formed afterwards. - if self.sparse_metadata_params.mtp_index_share: - # Slots are numbered by the bound manager; the draft loop binds the - # separate draft manager. - num_slots = len(self.kv_cache_manager.qsa_shared_topk_slots) - if self.qsa_draft_block_table is not None: - num_slots = max(num_slots, len(self.draft_kv_cache_manager.qsa_shared_topk_slots)) - self.qsa_shared_topk_indices = self.get_empty( - buffers, - ( - num_slots, - self.max_num_sequences, - self.sparse_metadata_params.block_topk, - ), - cache_name="qsa_shared_topk_indices", - dtype=torch.int32, - capture_graph=capture_graph, + if params.mtp_index_share: + # Slots are numbered by the bound manager (the draft manager on a + # draft metadata over a separate draft KV cache). + num_slots = len(md.kv_cache_manager.qsa_shared_topk_slots) + self.index_share.bind_buffers( + shared_topk_indices=md.get_buffer( + (num_slots, md.max_num_sequences, params.block_topk), + torch.int32, + "qsa_shared_topk_indices", + ) ) - self.qsa_shared_topk_visible_blocks = self.get_empty( - buffers, - (num_slots, self.max_num_sequences), - cache_name="qsa_shared_topk_visible_blocks", - dtype=torch.int32, - capture_graph=capture_graph, + self.qsa_shared_topk_visible_blocks = md.get_buffer( + (num_slots, md.max_num_sequences), + torch.int32, + "qsa_shared_topk_visible_blocks", ) - def _init_qsa_draft_block_table(self) -> None: - """Allocate the page table of a separate one-model draft KV cache. - - The draft layers' pages live in the draft manager's pools, which the - target table does not address. This table is its own graph-stable - buffer, so a captured graph reads the target table in the target - forward and this one in the draft loop; ``prepare()`` refreshes both - before every replay. - """ - from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import Qwen4ExpHybridCacheManager - - draft_manager = self.draft_kv_cache_manager - if ( - not isinstance(draft_manager, Qwen4ExpHybridCacheManager) - or draft_manager.qsa_position_layer_id is None - ): - return - self.qsa_draft_block_table = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences, draft_manager.max_blocks_per_seq), - cache_name="qsa_draft_block_table", - dtype=torch.int32, - capture_graph=self.is_cuda_graph, - ) - self._qsa_draft_attention_pool_id, self._qsa_draft_page_index_scale = ( - draft_manager.get_qsa_attention_pool_layout() - ) - - def set_skip_topk(self, skip: bool) -> None: - """Mark a draft step that reuses the captured selection.""" - self.qsa_indexer_skip_topk = skip - - def set_in_mtp_draft_loop(self, active: bool) -> None: - """Bracket the MTP draft loop so the indexer captures once and reuses.""" - self.qsa_in_mtp_draft_loop = active - # Readiness is per layer slot, not global: a layer may reuse only the - # row it captured itself, and the layers do not all capture in the same - # pass. A loop whose first step is a context pass captures nothing, so - # the next step captures layer by layer. - self.qsa_shared_topk_captured_slots.clear() - - def set_mtp_num_accepted(self, num_accepted: Optional[torch.Tensor]) -> None: - """Supply the per-request accepted counts used to pick the capture row.""" - self.qsa_mtp_num_accepted = num_accepted - - def _refresh_qsa_token_mapping(self) -> None: + def _refresh_token_mapping(self) -> None: """Map packed rows to requests, logical positions, and causal limits.""" - num_seqs = self.num_seqs - num_tokens = self.num_tokens + md = self.metadata + num_seqs = md.num_seqs + num_tokens = md.num_tokens if num_seqs <= 0 or num_tokens <= 0: return - seq_lens = self.seq_lens_cuda[:num_seqs] - if self.num_contexts == 0 and num_tokens == num_seqs: + seq_lens = md.seq_lens_cuda[:num_seqs] + if md.num_contexts == 0 and num_tokens == num_seqs: from .kernels import triton_qsa_decode_token_mapping triton_qsa_decode_token_mapping( - kv_lens=self.kv_lens_cuda_runtime[:num_seqs], + kv_lens=md.kv_lens_cuda_runtime[:num_seqs], seq_lens=seq_lens, - request_indices=self.qsa_req_idx_per_token[:num_tokens], + request_indices=self.index_share.req_idx_per_token[:num_tokens], logical_positions=self.qsa_logical_positions[:num_tokens], sequence_lengths=self.qsa_sequence_lengths[:num_tokens], visible_blocks=self.qsa_visible_blocks[:num_tokens], compress_ratio=self.sparse_metadata_params.compress_ratio, ) return - if self.num_contexts == 0 and num_seqs <= QSA_GENERATION_MAPPING_MAX_SEQS: + if md.num_contexts == 0 and num_seqs <= QSA_GENERATION_MAPPING_MAX_SEQS: triton_qsa_generation_token_mapping( - kv_lens=self.kv_lens_cuda_runtime[:num_seqs], + kv_lens=md.kv_lens_cuda_runtime[:num_seqs], seq_lens=seq_lens, - request_indices=self.qsa_req_idx_per_token, + request_indices=self.index_share.req_idx_per_token, logical_positions=self.qsa_logical_positions, sequence_lengths=self.qsa_sequence_lengths, visible_blocks=self.qsa_visible_blocks, @@ -276,27 +211,25 @@ def _refresh_qsa_token_mapping(self) -> None: seq_lens, dim=0, dtype=torch.int64, - out=self._qsa_cu_seq_lens[1 : num_seqs + 1], + out=self._cu_seq_lens[1 : num_seqs + 1], ) - token_ids = self._qsa_token_arange[:num_tokens] + token_ids = self._token_arange[:num_tokens] req_idx = torch.searchsorted( - self._qsa_cu_seq_lens[1 : num_seqs + 1], + self._cu_seq_lens[1 : num_seqs + 1], token_ids, right=True, ) - self.qsa_req_idx_per_token[:num_tokens].copy_(req_idx.to(torch.int32)) - seq_starts = self._qsa_cu_seq_lens[:num_seqs] - cached_lens = self.kv_lens_cuda_runtime[:num_seqs].to(torch.int64) - seq_lens.to( - torch.int64 - ) + self.index_share.req_idx_per_token[:num_tokens].copy_(req_idx.to(torch.int32)) + seq_starts = self._cu_seq_lens[:num_seqs] + cached_lens = md.kv_lens_cuda_runtime[:num_seqs].to(torch.int64) - seq_lens.to(torch.int64) logical = cached_lens[req_idx] + token_ids - seq_starts[req_idx] self.qsa_logical_positions[:num_tokens].copy_(logical) - self.qsa_sequence_lengths[:num_tokens].copy_(self.kv_lens_cuda_runtime[:num_seqs][req_idx]) + self.qsa_sequence_lengths[:num_tokens].copy_(md.kv_lens_cuda_runtime[:num_seqs][req_idx]) self.qsa_visible_blocks[:num_tokens].copy_( ((logical + 1) // self.sparse_metadata_params.compress_ratio).to(torch.int32) ) - def _unscale_qsa_block_table( + def _unscale_block_table( self, block_offsets: torch.Tensor, pool_id: int, @@ -304,7 +237,7 @@ def _unscale_qsa_block_table( block_table: torch.Tensor, ) -> None: """Decode one manager's V2 attention offsets into raw page IDs.""" - num_seqs = self.num_seqs + num_seqs = self.metadata.num_seqs if num_seqs <= 0: return triton_qsa_unscale_block_table( @@ -313,94 +246,51 @@ def _unscale_qsa_block_table( page_index_scale=page_index_scale, ) - def _refresh_qsa_block_table(self) -> None: - """Decode the bound manager's offsets into the table QSA kernels read. - - While ``prepare_for_draft_forward()`` is in effect every operand here is - the draft manager's, so this rebuilds the draft table. - """ - self._unscale_qsa_block_table( - self.kv_cache_block_offsets, - self._qsa_attention_pool_id, - self._qsa_page_index_scale, - self.qsa_block_table, - ) - - def _refresh_qsa_draft_block_table(self) -> None: - """Decode the separate draft manager's offsets into the draft table.""" - if self.qsa_draft_block_table is None: - return - self._unscale_qsa_block_table( - self.draft_kv_cache_block_offsets, - self._qsa_draft_attention_pool_id, - self._qsa_draft_page_index_scale, - self.qsa_draft_block_table, + def _refresh_block_table(self) -> None: + """Decode the bound manager's offsets into the table QSA kernels read.""" + self._unscale_block_table( + self.metadata.kv_cache_block_offsets, + self._attention_pool_id, + self._page_index_scale, + self.index_share.block_table, ) - def _refresh_qsa_speculative_snapshot_state(self) -> None: + def _refresh_speculative_snapshot_state(self) -> None: """Cache whether eager generation contains multi-token verification rows. Host lengths are already available during prepare; using them avoids a device reduction and synchronization in every QSA layer. """ - generation_lens = self.seq_lens[self.num_contexts : self.num_seqs] + md = self.metadata + generation_lens = md.seq_lens[md.num_contexts : md.num_seqs] self.qsa_needs_speculative_snapshot = any(length > 1 for length in generation_lens.tolist()) - def _refresh_qsa_device_state(self) -> None: + def _refresh_device_state(self) -> None: """Rebuild the device-side block table and token mapping for this step.""" if not self.qsa_has_local_layers: return - self._refresh_qsa_block_table() - self._refresh_qsa_draft_block_table() - self._refresh_qsa_token_mapping() - - def prepare_for_draft_forward(self) -> None: - """Point the QSA page table at the separate draft manager's pages. - - The caller has already swapped ``kv_cache_manager`` and - ``kv_cache_block_offsets`` to the draft manager's. The draft table was - decoded in ``prepare()``, so this only rebinds fields and launches - nothing; ``restore_draft_swaps()`` restores the target bindings. The - row mapping is manager-independent and stays as it is. - """ - if self.qsa_draft_block_table is None: - return - if self.kv_cache_manager is not self.draft_kv_cache_manager: - raise RuntimeError( - "QSA draft forward expects the metadata's draft KV cache manager " - f"to be bound; got {type(self.kv_cache_manager).__name__}" - ) - self.swap_for_draft("qsa_block_table", self.qsa_draft_block_table) - self.swap_for_draft("_qsa_attention_pool_id", self._qsa_draft_attention_pool_id) - self.swap_for_draft("_qsa_page_index_scale", self._qsa_draft_page_index_scale) + self._refresh_block_table() + self._refresh_token_mapping() def prepare(self) -> None: """Refresh host decisions, then rebuild graph-stable device mappings.""" - super().prepare() if not self.qsa_has_local_layers: return - self._refresh_qsa_speculative_snapshot_state() - self._refresh_qsa_device_state() + self._refresh_speculative_snapshot_state() + self._refresh_device_state() - def on_update_kv_lens(self) -> None: - """Refresh row limits after speculative decoding advances KV lengths.""" - super().on_update_kv_lens() - if not self.qsa_has_local_layers: - return - # Page allocation is unchanged within the step, so only the row mapping - # depends on the updated lengths. - self._refresh_qsa_token_mapping() + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + # A draft step also repacks the batch from one row per accepted token + # to a single row per request. + if self.qsa_has_local_layers: + self._refresh_token_mapping() - def update_for_spec_dec(self) -> None: - """Refresh the row mapping after a draft step repacks the batch. - The first draft step rewrites seq_lens from one row per accepted token - to a single row per request and advances kv_lens_cuda in place, so the - packed-row mapping built during prepare() no longer describes the rows - the next draft step attends over. - """ - super().update_for_spec_dec() - self.on_update_kv_lens() +class QSAAttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the QSA extension attached.""" + + backend_extension_types = (QSAMetadata,) -__all__ = ["QSAAttentionMetadata"] +__all__ = ["QSAAttentionMetadata", "QSAMetadata"] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/rocket/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/rocket/__init__.py index 85c3c9e33f3b..06cff387152f 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/rocket/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/rocket/__init__.py @@ -5,13 +5,14 @@ from .backend import RocketTrtllmAttention, RocketVanillaAttention from .cache_manager import RocketKVCacheManager -from .metadata import RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata +from .metadata import RocketMetadata, RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata from .params import RocketKVMetadataParams, RocketKVParams __all__ = [ "RocketKVCacheManager", "RocketKVMetadataParams", "RocketKVParams", + "RocketMetadata", "RocketTrtllmAttention", "RocketTrtllmAttentionMetadata", "RocketVanillaAttention", diff --git a/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py index 054c69855dbb..e4fb4ccf680e 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py @@ -10,8 +10,10 @@ import tensorrt_llm import tensorrt_llm.bindings +from tensorrt_llm._torch.attention.backends.sparse.params import SparseAlgorithmExtension from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata +from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._utils import prefer_pinned from .params import RocketKVMetadataParams @@ -19,108 +21,140 @@ ModelConfig = tensorrt_llm.bindings.ModelConfig -class RocketTrtllmAttentionMetadata(TrtllmAttentionMetadata): - def __post_init__(self): - super().__post_init__() - sparse_metadata_params = self.sparse_metadata_params - if not isinstance(sparse_metadata_params, RocketKVMetadataParams): - raise ValueError("RocketKV sparse attention metadata params are not set") - self.prompt_budget = sparse_metadata_params.prompt_budget - self.window_size = sparse_metadata_params.window_size - self.page_size = sparse_metadata_params.page_size - self.topk = sparse_metadata_params.topk +class RocketMetadata(SparseAlgorithmExtension): + """RocketKV state: the prompt budget, the KT cache and the index layouts. + + Serves the TRTLLM and Vanilla backends. On both, a generation request + attends to at most ``prompt_budget`` prompt tokens, so its cached length is + reduced before the KV lengths are derived. The TRTLLM backend additionally + predicts sparse indices from the per-step context and generation layouts + prepared here; the Vanilla backend predicts per request in Python. + """ + + fields = frozenset( + { + "context_cumsum", + "context_cumsum_cuda", + "cum_kt_lens", + "cum_kt_lens_cuda", + "host_kt_cache_block_offsets", + "k_context_lens", + "k_context_lens_cuda", + "k_context_start_cuda", + "k_cu_seqlens", + "k_cu_seqlens_cuda", + "kt_cache_block_offsets", + "kt_tokens_per_block", + "max_kt_tokens", + "max_rocket_k_ctx_len", + "num_kt_tokens", + "prompt_budget", + "q_cu_seqlens", + "q_cu_seqlens_cuda", + "rocket_page_size", + "rocket_window_size", + "sparse_offsets_ctx", + "sparse_offsets_ctx_cuda", + "sparse_offsets_gen", + "sparse_offsets_gen_cuda", + "topk", + "total_kt_tokens", + "total_rocket_k_ctx_tokens", + "total_sparse_ctx_indices", + "total_sparse_gen_indices", + "valid_batch_size", + "valid_seq_indices_cuda", + } + ) + methods = frozenset() - assert self.page_size == next_power_of_2(self.page_size), "Page size must be a power of 2" + @property + def _predicts_on_device(self) -> bool: + return isinstance(self.metadata, TrtllmAttentionMetadata) - capture_graph = self.is_cuda_graph + @property + def kt_tokens_per_block(self) -> Optional[int]: + """The number of KT tokens per block from the KV cache manager.""" + kv_cache_manager = self.metadata.kv_cache_manager + return kv_cache_manager.kt_tokens_per_block if kv_cache_manager is not None else None + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + buffers = metadata.cuda_graph_buffers + params = self.sparse_metadata_params + if not isinstance(params, RocketKVMetadataParams): + raise ValueError("RocketKV sparse attention metadata params are not set") + self.prompt_budget = params.prompt_budget + self.rocket_window_size = params.window_size + self.rocket_page_size = params.page_size + self.topk = params.topk + # (cached-length parameters prepare presented, the batch's own): a + # repeated prepare of the same batch reads the presented ones. + self._budget_inputs = None + md = self.metadata + if not self._predicts_on_device: + self.kt_cache_block_offsets = torch.empty( + [md.max_num_sequences, md.kv_cache_manager.max_kt_blocks_per_seq], + dtype=torch.int32, + device="cuda", + ) + self.host_kt_cache_block_offsets = torch.zeros_like( + self.kt_cache_block_offsets, + device="cpu", + pin_memory=prefer_pinned(), + ) + return - # Cumulative valid sequence lengths for query and key - self.q_cu_seqlens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="q_cu_seqlens_cuda", - capture_graph=capture_graph, + # The TRTLLM kernels read the RocketKV page size as the cache page size. + md.override_page_size(self.rocket_page_size) + assert self.rocket_page_size == next_power_of_2(self.rocket_page_size), ( + "Page size must be a power of 2" ) - self.q_cu_seqlens = torch.zeros_like( - self.q_cu_seqlens_cuda, device="cpu", dtype=torch.int32 - ) + capture_graph = md.is_cuda_graph + max_num_sequences = md.max_num_sequences - self.k_cu_seqlens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="k_cu_seqlens_cuda", - capture_graph=capture_graph, - ) - self.k_cu_seqlens = torch.zeros_like( - self.k_cu_seqlens_cuda, device="cpu", dtype=torch.int32 - ) + def device_buffer(shape, name, pool=buffers): + return md.get_empty( + pool, shape, dtype=torch.int32, cache_name=name, capture_graph=capture_graph + ) + + def host_mirror(device): + return torch.zeros_like(device, device="cpu", dtype=torch.int32) + + # Cumulative valid sequence lengths for query and key + self.q_cu_seqlens_cuda = device_buffer((max_num_sequences + 1,), "q_cu_seqlens_cuda") + self.q_cu_seqlens = host_mirror(self.q_cu_seqlens_cuda) + self.k_cu_seqlens_cuda = device_buffer((max_num_sequences + 1,), "k_cu_seqlens_cuda") + self.k_cu_seqlens = host_mirror(self.k_cu_seqlens_cuda) # Context length of RocketKV key for each valid sequence - self.k_context_lens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), - dtype=torch.int32, - cache_name="k_context_lens_cuda", - capture_graph=capture_graph, - ) - self.k_context_lens = torch.zeros_like( - self.k_context_lens_cuda, device="cpu", dtype=torch.int32 - ) + self.k_context_lens_cuda = device_buffer((max_num_sequences,), "k_context_lens_cuda") + self.k_context_lens = host_mirror(self.k_context_lens_cuda) # Start index of RocketKV key for each valid sequence - self.k_context_start_cuda = self.get_empty( - None, - (self.max_num_sequences,), - dtype=torch.int32, - cache_name="k_context_start_cuda", - capture_graph=capture_graph, + self.k_context_start_cuda = device_buffer( + (max_num_sequences,), "k_context_start_cuda", pool=None ) # Cumulative context lengths for each sequence - self.context_cumsum_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="context_cumsum_cuda", - capture_graph=capture_graph, - ) - self.context_cumsum = torch.zeros_like( - self.context_cumsum_cuda, device="cpu", dtype=torch.int32 - ) + self.context_cumsum_cuda = device_buffer((max_num_sequences + 1,), "context_cumsum_cuda") + self.context_cumsum = host_mirror(self.context_cumsum_cuda) # Sparse kv indices offsets for each sequence in context phase - self.sparse_offsets_ctx_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="sparse_offsets_ctx_cuda", - capture_graph=capture_graph, - ) - self.sparse_offsets_ctx = torch.zeros_like( - self.sparse_offsets_ctx_cuda, device="cpu", dtype=torch.int32 + self.sparse_offsets_ctx_cuda = device_buffer( + (max_num_sequences + 1,), "sparse_offsets_ctx_cuda" ) + self.sparse_offsets_ctx = host_mirror(self.sparse_offsets_ctx_cuda) # Valid sequence indices used in sparse kv indices prediction - self.valid_seq_indices_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences,), - dtype=torch.int32, - cache_name="valid_seq_indices_cuda", - capture_graph=capture_graph, - ) + self.valid_seq_indices_cuda = device_buffer((max_num_sequences,), "valid_seq_indices_cuda") # KT cache block offsets used in KT cache related kernels - self.kt_cache_block_offsets = self.get_empty( - self.cuda_graph_buffers, - [self.max_num_sequences, self.kv_cache_manager.max_kt_blocks_per_seq], - dtype=torch.int32, - cache_name="kt_cache_block_offsets", - capture_graph=capture_graph, + self.kt_cache_block_offsets = device_buffer( + [max_num_sequences, md.kv_cache_manager.max_kt_blocks_per_seq], + "kt_cache_block_offsets", ) - self.host_kt_cache_block_offsets = torch.zeros_like( self.kt_cache_block_offsets, device="cpu", @@ -128,95 +162,76 @@ def __post_init__(self): ) # Number of KT tokens for each sequence - self.num_kt_tokens = torch.empty( - self.max_num_sequences, - device="cpu", - dtype=torch.int32, - ) + self.num_kt_tokens = torch.empty(max_num_sequences, device="cpu", dtype=torch.int32) # Cumulative KT lengths for each sequence - self.cum_kt_lens_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="cum_kt_lens_cuda", - capture_graph=capture_graph, - ) - self.cum_kt_lens = torch.zeros_like(self.cum_kt_lens_cuda, device="cpu", dtype=torch.int32) + self.cum_kt_lens_cuda = device_buffer((max_num_sequences + 1,), "cum_kt_lens_cuda") + self.cum_kt_lens = host_mirror(self.cum_kt_lens_cuda) # Sparse attn indices offsets for each sequence in generation phase - self.sparse_offsets_gen_cuda = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences + 1,), - dtype=torch.int32, - cache_name="sparse_offsets_gen_cuda", - capture_graph=capture_graph, - ) - self.sparse_offsets_gen = torch.zeros_like( - self.sparse_offsets_gen_cuda, device="cpu", dtype=torch.int32 + self.sparse_offsets_gen_cuda = device_buffer( + (max_num_sequences + 1,), "sparse_offsets_gen_cuda" ) + self.sparse_offsets_gen = host_mirror(self.sparse_offsets_gen_cuda) # Maximum number of KT tokens - self.max_kt_tokens = (self.max_seq_len + self.page_size - 1) // self.page_size - - @property - def kt_tokens_per_block(self) -> Optional[int]: - """ - Returns the number of kt tokens per block from the KV cache manager. - """ - return ( - self.kv_cache_manager.kt_tokens_per_block if self.kv_cache_manager is not None else None - ) - - def prepare(self): - if self.kv_cache_manager is not None: - num_contexts = self.num_contexts - num_generations = self.num_generations - num_requests = num_contexts + num_generations - - for i in range(num_requests): - if i < num_contexts: - self.kv_cache_params.num_cached_tokens_per_seq[i] = 0 - else: - if self.prompt_lens[i] > self.prompt_budget: - self.kv_cache_params.num_cached_tokens_per_seq[i] += ( - self.prompt_budget - self.prompt_lens[i] - ) - - super().prepare() - - # Update prompt lens for sparse attention - if self.kv_cache_manager is not None: - _prompt_lens = self.prompt_lens.copy() - for i in range(num_requests): - if i >= num_contexts: - _prompt_lens[i] = min(_prompt_lens[i], self.prompt_budget) - _prompt_lens = torch.tensor(_prompt_lens, dtype=torch.int, device="cpu") - self.prompt_lens_cpu[: self.num_seqs].copy_(_prompt_lens) - self.prompt_lens_cuda[: self.num_seqs].copy_( - self.prompt_lens_cpu[: self.num_seqs], non_blocking=True - ) - self.prompt_lens_cuda_runtime = self.prompt_lens_cuda[: self.num_seqs] - self.prompt_lens_cpu_runtime = self.prompt_lens_cpu[: self.num_seqs] + self.max_kt_tokens = (md.max_seq_len + self.rocket_page_size - 1) // self.rocket_page_size + + def _budgeted_cached_token_lens(self, batch_params: KVCacheParams) -> list[int]: + """Each sequence's cached length, generation ones capped at the prompt budget.""" + md = self.metadata + num_contexts = md.num_contexts + cached = list(batch_params.num_cached_tokens_per_seq) + for i in range(num_contexts + md.num_generations): + if i < num_contexts: + cached[i] = 0 + elif md.prompt_lens[i] > self.prompt_budget: + cached[i] += self.prompt_budget - md.prompt_lens[i] + return cached - # for kt cache - self.kv_cache_manager.copy_kt_block_offsets( - self.request_ids, self.host_kt_cache_block_offsets + def prepare(self) -> None: + md = self.metadata + if not (self._predicts_on_device and md.kv_cache_manager is None): + # Attend to at most prompt_budget prompt tokens: the kernels see + # the capped cached lengths, the batch's parameters stay as given. + batch_params = md.kv_cache_params + if self._budget_inputs is not None and self._budget_inputs[0] is batch_params: + batch_params = self._budget_inputs[1] + md.override_cached_token_lens(self._budgeted_cached_token_lens(batch_params)) + self._budget_inputs = (md.kv_cache_params, batch_params) + if md.kv_cache_manager is not None: + md.kv_cache_manager.copy_kt_block_offsets( + md.request_ids, self.host_kt_cache_block_offsets ) - self.kt_cache_block_offsets[: self.num_seqs].copy_( - self.host_kt_cache_block_offsets[: self.num_seqs], non_blocking=True + self.kt_cache_block_offsets[: md.num_seqs].copy_( + self.host_kt_cache_block_offsets[: md.num_seqs], non_blocking=True ) + self._prepare_sparse_offsets() + + def _prepare_sparse_offsets(self) -> None: + if not self._predicts_on_device: + return + md = self.metadata + num_contexts = md.num_contexts + num_generations = md.num_generations + + if md.kv_cache_manager is not None: + # Generation requests attend to at most prompt_budget prompt tokens. + prompt_lens = md.prompt_lens.copy() + for i in range(num_contexts, num_contexts + num_generations): + prompt_lens[i] = min(prompt_lens[i], self.prompt_budget) + md.override_prompt_lens(prompt_lens) # -------------------------------- Context phase -------------------------------- - self.context_cumsum[1 : self.num_contexts + 1] = torch.cumsum( - self.prompt_lens_cpu[: self.num_contexts], dim=0 + self.context_cumsum[1 : num_contexts + 1] = torch.cumsum( + md.prompt_lens_cpu[:num_contexts], dim=0 ) - self.context_cumsum_cuda[: self.num_contexts + 1].copy_( - self.context_cumsum[: self.num_contexts + 1], non_blocking=True + self.context_cumsum_cuda[: num_contexts + 1].copy_( + self.context_cumsum[: num_contexts + 1], non_blocking=True ) # We need to filter out sequences that are too short to skip sparse kv indices prediction - valid_mask = self.prompt_lens_cpu[: self.num_contexts] >= self.prompt_budget + valid_mask = md.prompt_lens_cpu[:num_contexts] >= self.prompt_budget valid_seq_indices = torch.where(valid_mask)[0] invalid_seq_indices = torch.where(~valid_mask)[0] valid_batch_size = len(valid_seq_indices) @@ -224,23 +239,24 @@ def prepare(self): # Only consider sequences that are long enough for sparse kv indices prediction in context phase self.k_context_lens[:valid_batch_size] = ( - self.prompt_lens_cpu[valid_seq_indices] - self.window_size + md.prompt_lens_cpu[valid_seq_indices] - self.rocket_window_size ) self.k_context_lens_cuda[:valid_batch_size].copy_( self.k_context_lens[:valid_batch_size], non_blocking=True ) - sparse_counts_ctx = torch.zeros(self.num_contexts, dtype=torch.int32, device="cpu") + sparse_counts_ctx = torch.zeros(num_contexts, dtype=torch.int32, device="cpu") sparse_counts_ctx[valid_seq_indices] = self.prompt_budget - sparse_counts_ctx[invalid_seq_indices] = self.prompt_lens_cpu[invalid_seq_indices] + sparse_counts_ctx[invalid_seq_indices] = md.prompt_lens_cpu[invalid_seq_indices] - self.sparse_offsets_ctx[1 : self.num_contexts + 1] = torch.cumsum(sparse_counts_ctx, dim=0) - self.sparse_offsets_ctx_cuda[: self.num_contexts + 1].copy_( - self.sparse_offsets_ctx[: self.num_contexts + 1], non_blocking=True + self.sparse_offsets_ctx[1 : num_contexts + 1] = torch.cumsum(sparse_counts_ctx, dim=0) + self.sparse_offsets_ctx_cuda[: num_contexts + 1].copy_( + self.sparse_offsets_ctx[: num_contexts + 1], non_blocking=True ) self.q_cu_seqlens[: valid_batch_size + 1] = ( - torch.arange(valid_batch_size + 1, device="cpu", dtype=torch.int32) * self.window_size + torch.arange(valid_batch_size + 1, device="cpu", dtype=torch.int32) + * self.rocket_window_size ) self.q_cu_seqlens_cuda[: valid_batch_size + 1].copy_( self.q_cu_seqlens[: valid_batch_size + 1], non_blocking=True @@ -262,76 +278,45 @@ def prepare(self): self.total_rocket_k_ctx_tokens = 0 self.valid_batch_size = valid_batch_size - self.total_sparse_ctx_indices = self.sparse_offsets_ctx[self.num_contexts].item() + self.total_sparse_ctx_indices = self.sparse_offsets_ctx[num_contexts].item() # -------------------------------- Generation phase -------------------------------- - self.num_kt_tokens[: self.num_generations] = ( - self.kv_lens[self.num_contexts : self.num_seqs] + self.page_size - 1 - ) // self.page_size + self.num_kt_tokens[:num_generations] = ( + md.kv_lens[num_contexts : md.num_seqs] + self.rocket_page_size - 1 + ) // self.rocket_page_size - self.cum_kt_lens[1 : self.num_generations + 1] = torch.cumsum( - self.num_kt_tokens[: self.num_generations], dim=0 + self.cum_kt_lens[1 : num_generations + 1] = torch.cumsum( + self.num_kt_tokens[:num_generations], dim=0 ) - self.cum_kt_lens_cuda[: self.num_generations + 1].copy_( - self.cum_kt_lens[: self.num_generations + 1], non_blocking=True + self.cum_kt_lens_cuda[: num_generations + 1].copy_( + self.cum_kt_lens[: num_generations + 1], non_blocking=True ) - self.total_kt_tokens = self.num_generations * self.max_kt_tokens + self.total_kt_tokens = num_generations * self.max_kt_tokens topk_tensor = torch.tensor(self.topk, dtype=torch.int32) # Some sequences may have less than topk KT tokens # We need to use the minimum of topk and the number of KT tokens - sparse_counts_gen = torch.minimum(topk_tensor, self.num_kt_tokens[: self.num_generations]) + sparse_counts_gen = torch.minimum(topk_tensor, self.num_kt_tokens[:num_generations]) - self.sparse_offsets_gen[1 : self.num_generations + 1] = torch.cumsum( - sparse_counts_gen[: self.num_generations], dim=0 + self.sparse_offsets_gen[1 : num_generations + 1] = torch.cumsum( + sparse_counts_gen[:num_generations], dim=0 ) - self.sparse_offsets_gen_cuda[: self.num_generations + 1].copy_( - self.sparse_offsets_gen[: self.num_generations + 1], non_blocking=True + self.sparse_offsets_gen_cuda[: num_generations + 1].copy_( + self.sparse_offsets_gen[: num_generations + 1], non_blocking=True ) - self.total_sparse_gen_indices = self.topk * self.num_generations + self.total_sparse_gen_indices = self.topk * num_generations -class RocketVanillaAttentionMetadata(VanillaAttentionMetadata): - def __post_init__(self): - super().__post_init__() - sparse_metadata_params = self.sparse_metadata_params - if not isinstance(sparse_metadata_params, RocketKVMetadataParams): - raise ValueError("RocketKV sparse attention metadata params are not set") - self.prompt_budget = sparse_metadata_params.prompt_budget - self.kt_cache_block_offsets = torch.empty( - [self.max_num_sequences, self.kv_cache_manager.max_kt_blocks_per_seq], - dtype=torch.int32, - device="cuda", - ) - self.host_kt_cache_block_offsets = torch.zeros_like( - self.kt_cache_block_offsets, - device="cpu", - pin_memory=prefer_pinned(), - ) +class RocketTrtllmAttentionMetadata(TrtllmAttentionMetadata): + """TRTLLM attention metadata with the RocketKV extension attached.""" - def prepare(self) -> None: - super().prepare() - num_contexts = self.num_contexts - num_generations = self.num_generations - num_requests = num_contexts + num_generations + backend_extension_types = (RocketMetadata,) - for i in range(num_requests): - if i < num_contexts: - self.kv_cache_params.num_cached_tokens_per_seq[i] = 0 - else: - if self.prompt_lens[i] > self.prompt_budget: - self.kv_cache_params.num_cached_tokens_per_seq[i] += ( - self.prompt_budget - self.prompt_lens[i] - ) - - if self.kv_cache_manager is not None: - # for kt cache - self.kv_cache_manager.copy_kt_block_offsets( - self.request_ids, self.host_kt_cache_block_offsets - ) - self.kt_cache_block_offsets[: self.num_seqs].copy_( - self.host_kt_cache_block_offsets[: self.num_seqs], non_blocking=True - ) + +class RocketVanillaAttentionMetadata(VanillaAttentionMetadata): + """Vanilla attention metadata with the RocketKV extension attached.""" + + backend_extension_types = (RocketMetadata,) diff --git a/tensorrt_llm/_torch/attention/backends/trtllm.py b/tensorrt_llm/_torch/attention/backends/trtllm.py index d549c790c2fd..289d65195b2e 100644 --- a/tensorrt_llm/_torch/attention/backends/trtllm.py +++ b/tensorrt_llm/_torch/attention/backends/trtllm.py @@ -14,13 +14,13 @@ # limitations under the License. import contextlib -import functools +import copy +import dataclasses import math import os import weakref -from dataclasses import dataclass, field -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, FrozenSet, Iterator, - List, Optional, Tuple) +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, FrozenSet, Iterator, + List, Optional, Tuple, Type) import numpy as np import torch @@ -29,8 +29,6 @@ from tensorrt_llm.mapping import Mapping from ...model_config import ModelConfig - from ...speculative.interface import SpecMetadata - from ...speculative.spec_tree_manager import SpecTreeManager from .fmha.gqa_decode import GqaDecodeFmha, GqaDecodeSetup from .fmha.interface import Fmha from .sparse.dsa.metadata import TokenMajorGenView @@ -53,16 +51,20 @@ from ...utils import (compute_swizzled_sf_shape, get_global_attrs, get_model_extra_attrs, helix_local_len_tensor) from ..kernels.sage_quant import SageQuantized, sage_quantize +from .core import AttentionCoreMetadata +from .extensions import MetadataExtension +from .features import DraftMetadata, FlashMlaMetadata, HelixMetadata +from .features.spec_decoding import SpecDecodingMetadata from .fmha.fp4_mla import Fp4MlaFmha from .fmha.manager import FmhaManager from .fp4_mla import (can_fuse_fp4_mla_q_quant, load_fp4_mla_chunked_kv_cache, - populate_helix_spec_buffers, scatter_fp4_mla_kv_cache) + scatter_fp4_mla_kv_cache) from .fp4_mla.state import Fp4MlaState from .interface import (AttentionBackend, AttentionForwardArgs, AttentionInputType, AttentionMask, AttentionMetadata, - KVCacheParams, MLAParams, PositionalEmbeddingParams, - PredefinedAttentionMask, RopeParams, - merge_attention_forward_args) + BatchInputs, KVCacheParams, MLAParams, + PositionalEmbeddingParams, PredefinedAttentionMask, + RopeParams, merge_attention_forward_args) from .sparse.hooks import prepare_sparse_runtime_params from .sparse.params import BlockSparseForwardInputs, SparseParams from .sparse.skip_softmax import SkipSoftmaxParams @@ -121,34 +123,7 @@ class _ContextPlan: "counter_rows", "total_num_blocks", "max_q_len", "max_kv_len") -@functools.cache -def generate_spec_decoding_position_offsets(max_num_requests: int, - draft_len: int) -> torch.Tensor: - width = draft_len + 1 - row = torch.arange(width, dtype=torch.int, device='cuda') - return row.unsqueeze(0).expand(max_num_requests, -1).contiguous() - - -@functools.cache -def generate_spec_decoding_packed_mask(max_num_requests: int, - draft_len: int) -> torch.Tensor: - width = draft_len + 1 - num_blocks = math.ceil(width / 32) - mask = torch.zeros([max_num_requests, width, num_blocks], - dtype=torch.int, - device='cuda') - remaining = width - for blk in range(num_blocks): - if remaining <= 0: - break - n = min(32, remaining) - vals = (torch.pow(2, torch.arange(n) + 1) - 1).int() - mask[:, blk * 32:blk * 32 + n, blk] = vals - remaining -= 32 - return mask - - -@dataclass(frozen=True, kw_only=True) +@dataclasses.dataclass(frozen=True, kw_only=True) class MlaLatentWriteTarget: """Where ``mla_rope_generation`` stores each token's new latent row. @@ -168,7 +143,7 @@ class MlaLatentWriteTarget: write_slots: torch.Tensor -@dataclass(frozen=True) +@dataclasses.dataclass(frozen=True) class HelixMlaChunkPlan: """CP-local read ranges for one global MLA history iteration. @@ -185,259 +160,259 @@ class HelixMlaChunkPlan: max_local_seq_len: int -@dataclass(kw_only=True) -class TrtllmAttentionMetadata(AttentionMetadata): - workspace: Optional[torch.Tensor] = None - cuda_graph_workspace: Optional[torch.Tensor] = None - workspace_reclaimable: bool = field(default=True, init=False) - # batch_block_table_key of the target / draft KV cache manager at the last - # block-offset copy of prepare_steady_decode_step or of a prepare() keyed - # by key_prepared_block_offsets; prepare() clears them. - _steady_block_table_keys: tuple = field(default=(None, None), - init=False, - repr=False, - compare=False) - # Whether the last prepare() copied the full block-offset table width. - _prepared_full_block_offsets: bool = field(default=False, - init=False, - repr=False, - compare=False) - - # TrtllmAttention needs to know the beam width to access to the cache indirection buffer, - # when beam search is enabled. - beam_width: int = 1 - - # Plan caches of the FMHA libraries, keyed by library. A planned wrapper owns - # workspaces that CUDA graphs capture, so it lives exactly as long as this - # metadata; every layer that runs with this metadata reuses it. - fmha_plan_caches: Dict[str, dict] = field(default_factory=dict, - init=False, - repr=False) - # Context-only plan shared by the layers of one forward (TrtllmAttention.cache_context_plan). - context_plan: Optional[_ContextPlan] = field(init=False, - default=None, - repr=False, - compare=False) - - @property - def effective_beam_width(self) -> int: - # Only use this for the fallback kernel's beam_width argument. - # Cross-attention reads request-scoped encoder K/V that is written once - # and reused unchanged by every decoder beam. Metadata preparation still - # uses beam_width to expand cross block-offset rows to decoder-sequence - # scope, but the fallback kernel should treat the cross K/V cache as - # non-beam-packed. - return 1 if self.is_cross else self.beam_width - - # TrtllmAttention needs to know the max sequence length. - # Implemented as a property to support no cache mode. - max_seq_len: Optional[int] - - # Storage for internal max_seq_len value - _max_seq_len_storage: Optional[int] = field(default=None, - init=True, - repr=False) - _page_size_override: Optional[int] = field(init=False, - default=None, - repr=False) - - # Encoder CUDA graph compatibility: overrides host-side max_context_q_len - # so FMHA kernel launch params are stable across graph capture/replay even - # when actual per-batch sequence lengths vary. Only set by - # EncoderCUDAGraphRunner; None elsewhere. - max_context_q_len_override: Optional[int] = None - - # Flags to enable spec-dec mode (multi-query mode) in TRTLLM XQA Kernels - # spec decoding mode can be enabled for non-TRTLLM-gen kernels (pre-Blackwell XQA kernels) - # is_spec_decoding_enabled specifies if spec-dec mode is supported for the entire runtime. - is_spec_decoding_enabled: bool = False - # use_spec_decoding determines if the attention layer should be run in spec-dec mode at the specific step / layer. - use_spec_decoding: bool = False - - # if spec-dec tree is a tree or a chain (linear tree) - is_spec_dec_tree: bool = False - # if spec-dec tree wouldn't be changed at all, the mask won't be computed every step. - is_spec_dec_dynamic_tree: bool = False - force_prepare_spec_dec_tree_mask: bool = False - - # parameters required for spec-dec mode - max_total_draft_tokens: Optional[int] = None - spec_decoding_position_offsets: Optional[torch.Tensor] = None - # Current query width of the compact prefix in a 1-D dynamic-tree offsets buffer. - spec_decoding_query_len: int = 0 - spec_decoding_packed_mask: Optional[torch.Tensor] = None - spec_decoding_generation_lengths: Optional[torch.Tensor] = None - spec_decoding_bl_tree_mask_offset: Optional[torch.Tensor] = None - spec_decoding_bl_tree_mask: Optional[torch.Tensor] = None - spec_bl_tree_first_sparse_mask_offset_kv: Optional[torch.Tensor] = None - - # TRTLLM-Gen FMHA JIT warmup controls. - trtllm_gen_jit_warmup: bool = False - - # Flag to enable helix parallelism. - enable_helix: bool = False - - # Global position ids of tokens for each sequence in the batch. Given - # each helix rank has only a subset of tokens for a sequence, we compute - # a global position id for each token here. - helix_position_offsets: Optional[torch.Tensor] = None - helix_position_offsets_cpu: Optional[torch.Tensor] = None - - # Whether the current rank is inactive for helix parallelism. - # In helix parallelism, only the active rank appends KV cache for the query token - # and attends to the previously cached tokens as well as the query token. Inactive - # ranks do not append KV cache for the query token and attend to the previously - # cached tokens only. - helix_is_inactive_rank: Optional[torch.Tensor] = None - helix_is_inactive_rank_cpu: Optional[torch.Tensor] = None - - # Int32 slots in each request's rank-local KV cache, one per context - # token. -1 skips persistence; temporary Q/K still cover every token. - helix_context_local_slots: Optional[torch.Tensor] = None - helix_mla_chunk_plans: list[HelixMlaChunkPlan] | None = None - - # Per-token helix state for speculative verify groups (a 1 + draft_len - # group may straddle a ledger-page boundary onto two CP ranks, so the - # per-sequence boolean above is insufficient there). See - # recompute_helix_spec_buffers for the derivation. - helix_local_slots: Optional[torch.Tensor] = None - helix_kv_bounds: Optional[torch.Tensor] = None - helix_owned_new_tokens_cpu: Optional[torch.Tensor] = None - _helix_spec_tokens_valid: bool = False - - # Block offsets for the target and draft KV caches - kv_cache_block_offsets: Optional[torch.Tensor] = None - host_kv_cache_block_offsets: Optional[torch.Tensor] = None - draft_kv_cache_block_offsets: Optional[torch.Tensor] = None - # Active block-ID buffers, defaulting to the target cache. Separate draft - # storage lets CUDA graphs bind stable target and draft addresses. - block_ids_per_seq: Optional[torch.Tensor] = None - kv_block_ids_per_seq: Optional[torch.Tensor] = None - draft_block_ids_per_seq: Optional[torch.Tensor] = None - draft_kv_block_ids_per_seq: Optional[torch.Tensor] = None - # Originals of attributes swapped for draft replay, recorded by - # swap_for_draft; the strict buffer check validates these instead. - draft_replay_swapped_attrs: Dict[str, Any] = field(default_factory=dict, - init=False, - repr=False, - compare=False) - # Tensors produced and consumed inside the captured graph; the strict - # buffer check skips them. - graph_temporary_attrs: ClassVar[FrozenSet[str]] = frozenset() - - # Batch-shared FP4 state; other attention paths allocate none of it. - fp4_mla_state: Optional[Fp4MlaState] = field(init=False, - default=None, - repr=False, - compare=False) - - # Pre-computed FlashMLA tile-scheduler metadata and num_splits. - # Computed once per forward pass in TrtllmAttention.forward() and reused across layers. - flash_mla_tile_scheduler_metadata: Optional[torch.Tensor] = None - flash_mla_num_splits: Optional[torch.Tensor] = None - _flash_mla_metadata_valid: bool = field(default=False, - init=False, - repr=False) - - use_paged_context_fmha: bool = field(init=False, default=False, repr=False) - - # FMHA prologue buffers for the MLA generation path, hoisted out of the per-layer - # RoPE kernel. `mla_cu_q_rows` counts Q ROWS (cumsum(q_lens) * num_heads); the two - # cu_kv/ctx buffers count KV lengths and context TOKENS respectively. - mla_cu_q_rows: Optional[torch.Tensor] = None - mla_cu_kv_seqlens: Optional[torch.Tensor] = None - mla_ctx_cu_q_seqlens: Optional[torch.Tensor] = None - _mla_scheduler_buffers_valid: bool = field(default=False, - init=False, - repr=False) - _mla_ctx_cu_seqlens_valid: bool = field(default=False, - init=False, - repr=False) - # CuTe DSL MLA decode plans its per-request split-KV counts once per step - # (cute_dsl_mla.py); the plan buffer and the key that marks it current. - _cute_dsl_mla_split_plan: Optional[torch.Tensor] = field(init=False, - default=None, - repr=False) - _cute_dsl_mla_split_plan_key: Optional[tuple] = field(init=False, - default=None, - repr=False) - - # `DSAtrtllmAttentionMetadata` overrides this; the dense path keeps 0. - num_sparse_topk: int = 0 - - @property - def effective_workspace(self) -> Optional[torch.Tensor]: - """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" - return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace - - @property - def max_context_length(self) -> int: - """ - Upper bound for a single context window. - Required max_seq_len for context-only attention cases like visual gen - """ - return min(self.max_seq_len, self.max_num_tokens) - - @property - def max_seq_len(self) -> int: - """ - Returns the max sequence length. - If the attention uses KV cache, it will return max_seq_len from the KV cache manager. - If the attention is no cache, max_seq_len should be set manually by user. - """ - if self.kv_cache_manager is not None: - return self.kv_cache_manager.max_seq_len - else: - assert self._max_seq_len_storage is not None, "max_seq_len should be set for no kv cache attention" - return self._max_seq_len_storage - - @max_seq_len.setter - def max_seq_len(self, value: int) -> None: - """ - Set the max sequence length for no cache attention. - """ - self._max_seq_len_storage = value - - @property - def tokens_per_block(self) -> Optional[int]: - """ - Returns the number of tokens per block from the KV cache manager. - """ - return self.kv_cache_manager.tokens_per_block if self.kv_cache_manager is not None else None +class TrtllmCoreMetadata(AttentionCoreMetadata): + """TRTLLM attention state: device lengths, block offsets, workspaces and MLA buffers. - @property - def page_size(self) -> int: - """ - Number of tokens per cache page. - """ - page_size_override = getattr(self, '_page_size_override', None) - if page_size_override is not None: - return page_size_override - assert self.kv_cache_manager is not None, "page_size requires a KV cache manager" - return self.kv_cache_manager.tokens_per_block - - @page_size.setter - def page_size(self, value: int) -> None: - self._page_size_override = value - - @property - def host_kv_cache_pool_pointers(self) -> Optional[torch.Tensor]: - """ - Returns the host KV cache pool pointers from the KV cache manager if KV cache manager is not None. - """ - return self.kv_cache_manager.kv_cache_pool_pointers if self.kv_cache_manager is not None else None - - @property - def host_kv_cache_pool_mapping(self) -> Optional[torch.Tensor]: - """ - Returns the host KV cache pool mapping from the KV cache manager if KV cache manager is not None. - """ - return self.kv_cache_manager.kv_cache_pool_mapping if self.kv_cache_manager is not None else None + ``allocate`` binds the graph-stable buffers on creation and on every + metadata copy; ``derive`` publishes the step's lengths, block offsets and + the runtime views the attention op reads. + """ - def __post_init__(self) -> None: - super().__post_init__() - self.enable_helix = self.mapping.has_cp_helix( - ) if self.mapping is not None else False + pages_by_block_offsets = True + # Every CUDA graph copy runs on the same workspaces. + shared = frozenset({"workspace", "cuda_graph_workspace"}) + adopts_target_state = True + # A draft binds the target's derived lengths, block offsets and context + # buffers. Its loop advances the device KV lengths, turns its context + # rows into generation rows and stages the FlashMLA block table in place; + # its runtime views slice its own buffers, and the MLA prefixes and split + # plan, with the flags saying whether they are current, are its own, + # built lazily from its own lengths. + draft_bound = frozenset({ + "_max_seq_len_storage", + "_page_size_override", + "_prepared_full_block_offsets", + "chunked_global_offset", + "chunked_loop_num", + "chunked_seq_len", + "ctx_cached_token_indptr", + "ctx_kv_indptr", + "ctx_uncached_token_indptr", + "cu_chunked_seq_len", + "host_chunked_global_offset", + "host_chunked_seq_len", + "host_ctx_cached_token_indptr", + "host_ctx_kv_indptr", + "host_ctx_uncached_token_indptr", + "host_cu_chunked_seq_len", + "host_merge_op_tensor", + "host_total_kv_lens", + "helix_context_local_slots", + "helix_mla_chunk_plans", + "kv_block_ids_per_seq", + "kv_cache_block_offsets", + "kv_lens", + "kv_lens_runtime", + "max_chunk_len_per_loop", + "max_context_q_len_override", + "max_ctx_cached_token_len", + "max_ctx_kv_len", + "max_ctx_seq_len", + "merge_op_tensor", + "num_ctx_cached_tokens", + "num_sparse_topk", + "prompt_lens_cpu", + "prompt_lens_cpu_runtime", + "prompt_lens_cuda", + "prompt_lens_cuda_runtime", + "trtllm_gen_jit_warmup", + "use_paged_context_fmha", + "workspace_reclaimable", + }) + draft_written = frozenset( + {"kv_lens_cuda", "host_request_types", "block_ids_per_seq"}) + draft_private = frozenset({ + "_mla_scheduler_buffers_valid", + "_mla_ctx_cu_seqlens_valid", + "_cute_dsl_mla_split_plan_key", + "host_kv_cache_block_offsets", + "kv_lens_cuda_runtime", + "host_request_types_runtime", + "workspace", + "cuda_graph_workspace", + "fmha_plan_caches", + "mla_cu_q_rows", + "mla_cu_kv_seqlens", + "mla_ctx_cu_q_seqlens", + "_cute_dsl_mla_split_plan", + "context_plan", + "_steady_block_table_key", + "_aligned_kv_lens_staging", + "_context_scratch_helix", + }) + + fields = frozenset({ + "block_ids_per_seq", + "chunked_global_offset", + "chunked_loop_num", + "chunked_seq_len", + "ctx_cached_token_indptr", + "ctx_kv_indptr", + "context_plan", + "ctx_uncached_token_indptr", + "cu_chunked_seq_len", + "cuda_graph_workspace", + "effective_beam_width", + "effective_workspace", + "fmha_plan_caches", + "host_chunked_global_offset", + "host_chunked_seq_len", + "host_ctx_cached_token_indptr", + "host_ctx_kv_indptr", + "host_ctx_uncached_token_indptr", + "host_cu_chunked_seq_len", + "host_kv_cache_block_offsets", + "host_kv_cache_pool_mapping", + "host_kv_cache_pool_pointers", + "host_merge_op_tensor", + "host_request_types", + "host_request_types_runtime", + "host_total_kv_lens", + "helix_context_local_slots", + "helix_mla_chunk_plans", + "kv_block_ids_per_seq", + "kv_cache_block_offsets", + "kv_lens", + "kv_lens_cuda", + "kv_lens_cuda_runtime", + "kv_lens_runtime", + "max_chunk_len_per_loop", + "max_context_length", + "max_context_q_len_override", + "max_ctx_cached_token_len", + "max_ctx_kv_len", + "max_ctx_seq_len", + "max_seq_len", + "merge_op_tensor", + "mla_ctx_cu_q_seqlens", + "mla_cu_kv_seqlens", + "mla_cu_q_rows", + "num_ctx_cached_tokens", + "num_sparse_topk", + "page_size", + "prompt_lens_cpu", + "prompt_lens_cpu_runtime", + "prompt_lens_cuda", + "prompt_lens_cuda_runtime", + "tokens_per_block", + "trtllm_gen_jit_warmup", + "use_paged_context_fmha", + "workspace", + "workspace_reclaimable", + }) + methods = frozenset({ + "allocate_context_kv_indptrs", + "bind_context_tile", + "bind_fixed_kv_window", + "bind_generation_tile", + "bind_runtime_views", + "aligned_kv_lens_staging", + "bind_encoder_cuda_graph_seq_lens", + "cache_context_plan", + "clear_context_cached_kv", + "cute_dsl_mla_split_plan", + "disable_workspace_reclamation", + "mark_trtllm_gen_jit_warmup", + "override_page_size", + "override_prompt_lens", + "republish_kv_lens", + "context_scratch_view", + "set_num_sparse_topk", + "set_use_paged_context_fmha", + "stage_context_cached_kv", + "substitute_runtime_views", + "use_block_id_tables", + "use_block_offsets", + "use_workspace", + "reallocate_workspace", + "mla_prepare_ctx_cu_seqlens", + "mla_prepare_scheduler_buffers", + "pre_process_for_chunked_prefill", + "prepare_context_mla_with_cached_kv", + "prepare_encoder_cuda_graph_replay", + "prepare_encoder_decoder_from_precomputed_lengths", + "key_prepared_block_offsets", + "prepare_steady_decode_step", + "supports_steady_decode_step", + "presented_token_major", + "token_major_view", + }) + init_fields = { + "workspace": None, + "cuda_graph_workspace": None, + # The sequence bound of no-cache attention; a cache's comes from its + # manager. + "max_seq_len": None, + # Encoder CUDA graph compatibility: overrides host-side max_context_q_len + # so FMHA kernel launch params are stable across graph capture/replay even + # when actual per-batch sequence lengths vary. Only set by + # EncoderCUDAGraphRunner; None elsewhere. + "max_context_q_len_override": None, + # TRTLLM-Gen FMHA JIT warmup controls. + "trtllm_gen_jit_warmup": False, + # Block offsets of the KV cache, device and host. + "kv_cache_block_offsets": None, + "host_kv_cache_block_offsets": None, + # Per-sequence block IDs: generation rows (compacted) and every row. None + # unless an extension that needs them (FlashMLA) provides them. + "block_ids_per_seq": None, + "kv_block_ids_per_seq": None, + # FMHA prologue buffers for the MLA generation path, hoisted out of the per-layer + # RoPE kernel. `mla_cu_q_rows` counts Q ROWS (cumsum(q_lens) * num_heads); the two + # cu_kv/ctx buffers count KV lengths and context TOKENS respectively. + "mla_cu_q_rows": None, + "mla_cu_kv_seqlens": None, + "mla_ctx_cu_q_seqlens": None, + # The DSA extension sets this; the dense path keeps 0. + "num_sparse_topk": 0, + } + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.workspace_reclaimable = True + # Plan caches of the FMHA libraries, keyed by library. A planned wrapper owns + # workspaces that CUDA graphs capture, so it lives exactly as long as this + # metadata; every layer that runs with this metadata reuses it. + self.fmha_plan_caches = {} + # Context-only plan shared by the layers of one forward + # (TrtllmAttention.cache_context_plan); dropped whenever the lengths or + # the batch can change. The plan also compares its bound tensors by + # identity. + self.context_plan: Optional[_ContextPlan] = None + # batch_block_table_key of the KV cache manager at the last block-offset + # copy of prepare_steady_decode_step or of a prepare keyed by + # key_prepared_block_offsets; derive clears it. + self._steady_block_table_key = None + # Whether the last derive copied the full block-offset table width. + self._prepared_full_block_offsets = False + # Helix prefill: int32 slots in each request's rank-local KV cache, + # one per context token (-1 skips persistence; temporary Q/K still + # cover every token), and the CP-local read ranges of each chunked + # MLA history iteration. None without Helix cached-KV context MLA. + self.helix_context_local_slots: Optional[torch.Tensor] = None + self.helix_mla_chunk_plans: Optional[List[HelixMlaChunkPlan]] = None + # Kernel-aligned staging copy of misaligned device KV lengths + # (aligned_kv_lens_staging), built on first use. + self._aligned_kv_lens_staging: Optional[torch.Tensor] = None + # The Helix state context_scratch_view presents positions through when + # no Helix is attached; built on first use. + self._context_scratch_helix: Optional[HelixMetadata] = None + # Storage behind max_seq_len, which the base constructor may have set + # already (no-cache attention). + self.__dict__.setdefault("_max_seq_len_storage", None) + self._page_size_override = None + self.use_paged_context_fmha = False + self._mla_scheduler_buffers_valid = False + self._mla_ctx_cu_seqlens_valid = False + # CuTe DSL MLA decode plans its per-request split-KV counts once per step + # (cute_dsl_mla.py); the plan buffer and the key that marks it current. + self._cute_dsl_mla_split_plan = None + self._cute_dsl_mla_split_plan_key = None + # Every CUDA graph copy runs on its source's workspaces. + self.cuda_graph_workspace = self.shared_or( + "cuda_graph_workspace", lambda: self.cuda_graph_workspace) + self.workspace = self.shared_or("workspace", lambda: self.workspace) + buffers = metadata.cuda_graph_buffers self.use_paged_context_fmha = ( self.runtime_features.chunked_prefill or self.runtime_features.cache_reuse @@ -446,19 +421,16 @@ def __post_init__(self) -> None: # CUDA-graph metadata is a shallow copy that re-runs this method; give it # its own plan caches so each captured batch size plans its own wrappers. self.fmha_plan_caches = {} - # Each copy records and restores its own draft swaps. - self.draft_replay_swapped_attrs = {} - self._post_init_with_buffers(self.cuda_graph_buffers) + self._allocate_buffers(buffers) - def _post_init_with_buffers(self, buffers) -> None: - - # Set a default value, as max_num_sequences is not always set. + def _allocate_buffers(self, buffers) -> None: + md = self.metadata if self.max_num_sequences is None: self.max_num_sequences = self.max_num_requests capture_graph = self.is_cuda_graph - self.prompt_lens_cuda = self.get_empty( + self.prompt_lens_cuda = md.get_empty( buffers, (self.max_num_sequences, ), cache_name="prompt_lens_cuda", @@ -470,7 +442,7 @@ def _post_init_with_buffers(self, buffers) -> None: device='cpu', pin_memory=prefer_pinned(), ) - self.kv_lens_cuda = self.get_empty_like( + self.kv_lens_cuda = md.get_empty_like( buffers, self.prompt_lens_cuda, cache_name="kv_lens_cuda", @@ -483,14 +455,14 @@ def _post_init_with_buffers(self, buffers) -> None: self.host_request_types = torch.empty_like(self.prompt_lens_cpu) # `+ 1` because these are exclusive-prefix arrays over num_seqs. - self.mla_cu_q_rows = self.get_empty( + self.mla_cu_q_rows = md.get_empty( buffers, (self.max_num_sequences + 1, ), cache_name="mla_cu_q_rows", dtype=torch.int, capture_graph=capture_graph, ) - self.mla_cu_kv_seqlens = self.get_empty( + self.mla_cu_kv_seqlens = md.get_empty( buffers, (self.max_num_sequences + 1, ), cache_name="mla_cu_kv_seqlens", @@ -499,7 +471,7 @@ def _post_init_with_buffers(self, buffers) -> None: ) # Own copy because `ctx_uncached_token_indptr` holds the same values but only # exists under `enable_context_mla_with_cached_kv`. - self.mla_ctx_cu_q_seqlens = self.get_empty( + self.mla_ctx_cu_q_seqlens = md.get_empty( buffers, (self.max_num_sequences + 1, ), cache_name="mla_ctx_cu_q_seqlens", @@ -531,7 +503,7 @@ def _post_init_with_buffers(self, buffers) -> None: num_attention_op_pools = getattr(self.kv_cache_manager, "num_attention_op_pools", self.kv_cache_manager.num_pools) - self.kv_cache_block_offsets = self.get_empty( + self.kv_cache_block_offsets = md.get_empty( buffers, [ num_attention_op_pools, self.max_num_sequences, 2, @@ -544,101 +516,11 @@ def _post_init_with_buffers(self, buffers) -> None: self.host_kv_cache_block_offsets = self.kv_cache_manager.host_kv_cache_block_offsets self.block_ids_per_seq = None self.kv_block_ids_per_seq = None - self.draft_block_ids_per_seq = None - self.draft_kv_block_ids_per_seq = None - - # Allocate separate block offset tensors for draft KV cache manager - # Used in one-model speculative decoding with different KV cache layouts - if self.draft_kv_cache_manager is not None: - num_draft_attention_op_pools = getattr( - self.draft_kv_cache_manager, "num_attention_op_pools", - self.draft_kv_cache_manager.num_pools) - self.draft_kv_cache_block_offsets = self.get_empty( - buffers, - [ - num_draft_attention_op_pools, self.max_num_sequences, 2, - self.draft_kv_cache_manager.max_blocks_per_seq - ], - cache_name="draft_kv_cache_block_offsets", - dtype=torch.int32, - capture_graph=capture_graph, - ) - if self.enable_flash_mla: - self.block_ids_per_seq = self.get_empty( - buffers, - [ - self.kv_cache_manager.max_batch_size, - self.kv_cache_manager.max_blocks_per_seq - ], - cache_name="block_ids_per_seq", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.kv_block_ids_per_seq = self.get_empty( - buffers, - [ - self.kv_cache_manager.max_batch_size, - self.kv_cache_manager.max_blocks_per_seq - ], - cache_name="kv_block_ids_per_seq", - dtype=torch.int32, - capture_graph=capture_graph, - ) - if self.draft_kv_cache_manager is not None: - self.draft_block_ids_per_seq = self.get_empty( - buffers, - [ - self.draft_kv_cache_manager.max_batch_size, - self.draft_kv_cache_manager.max_blocks_per_seq - ], - cache_name="draft_block_ids_per_seq", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.draft_kv_block_ids_per_seq = self.get_empty( - buffers, - [ - self.draft_kv_cache_manager.max_batch_size, - self.draft_kv_cache_manager.max_blocks_per_seq - ], - cache_name="draft_kv_block_ids_per_seq", - dtype=torch.int32, - capture_graph=capture_graph, - ) - # Allocate fixed-size buffers for pre-computed FlashMLA metadata. - # These are pre-allocated so their GPU addresses are stable across CUDA graph captures. - sm_count = torch.cuda.get_device_properties( - torch.cuda.current_device()).multi_processor_count - self.flash_mla_tile_scheduler_metadata = self.get_empty( - buffers, - [sm_count * 8], # TileSchedulerMetaDataSize = 8 - cache_name="flash_mla_tile_scheduler_metadata", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.flash_mla_num_splits = self.get_empty( - buffers, - [self.kv_cache_manager.max_batch_size + 1], - cache_name="flash_mla_num_splits", - dtype=torch.int32, - capture_graph=capture_graph, - ) if self.enable_context_mla_with_cached_kv: # for kv cache reuse/chunked context in MLA - self.ctx_cached_token_indptr = self.get_empty( - buffers, - (self.max_num_requests + 1, ), - cache_name="ctx_cached_token_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_cached_token_indptr = torch.zeros_like( - self.ctx_cached_token_indptr, - device='cpu', - pin_memory=prefer_pinned(), - ) - self.ctx_uncached_token_indptr = self.get_empty( + self.allocate_context_kv_indptrs(buffers, capture_graph) + self.ctx_uncached_token_indptr = md.get_empty( buffers, (self.max_num_requests + 1, ), cache_name="ctx_uncached_token_indptr", @@ -650,163 +532,338 @@ def _post_init_with_buffers(self, buffers) -> None: device='cpu', pin_memory=prefer_pinned(), ) - # context full seqlens include cached tokens and uncached tokens - self.ctx_kv_indptr = self.get_empty( - buffers, - (self.max_num_requests + 1, ), - cache_name="ctx_kv_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_kv_indptr = torch.zeros_like( - self.ctx_kv_indptr, - device='cpu', - pin_memory=prefer_pinned(), - ) - if callable( - getattr(self.kv_cache_manager, "get_fp4_mla_page_table_spec", - None)): - self.fp4_mla_state = Fp4MlaState.create(self, buffers) + def allocate_context_kv_indptrs(self, buffers, capture_graph: bool) -> None: + """Allocate the context cached-token and KV-length prefix buffers. - # Allocate static buffers for helix parallelism support. - if self.enable_helix: - self.helix_position_offsets = self.get_empty( - buffers, - (self.max_num_tokens, ), - cache_name="helix_position_offsets", - dtype=torch.int, - capture_graph=capture_graph, - ) - self.helix_position_offsets_cpu = torch.empty_like( - self.helix_position_offsets, - device='cpu', - pin_memory=prefer_pinned(), - ) - self.helix_is_inactive_rank = self.get_empty( - buffers, - (self.max_num_sequences, ), - cache_name="helix_is_inactive_rank", - dtype=torch.bool, - capture_graph=capture_graph, - ) - self.helix_is_inactive_rank_cpu = torch.empty_like( - self.helix_is_inactive_rank, - device='cpu', - pin_memory=prefer_pinned(), - ) - # Per-token buffers for speculative verify groups under helix. - # A group of 1 + draft_len tokens can straddle a ledger-page - # boundary, splitting ownership between two CP ranks, so the - # per-sequence flag above is not expressive enough: - # helix_local_slots[t]: rank-local KV write slot of gen token t - # on this rank, or -1 when another rank owns its position - # (consumed by the mla_rope_generation append kernel). - # helix_kv_bounds[t]: number of rank-local KV entries token t - # may attend to, i.e. local_len(pos_t + 1) (consumed by the - # CuTe DSL MLA decode mask and the helix stats identity). - # Filled by recompute_helix_spec_buffers() on the spec path only. - self.helix_local_slots = self.get_empty( - buffers, - (self.max_num_tokens, ), - cache_name="helix_local_slots", - dtype=torch.int, - capture_graph=capture_graph, - ) - self.helix_kv_bounds = self.get_empty( - buffers, - (self.max_num_tokens, ), - cache_name="helix_kv_bounds", - dtype=torch.int, - capture_graph=capture_graph, - ) - # Host-side per-sequence count of this step's new tokens owned by - # this rank (spec path; single-token path derives it from the - # boolean flag). Consumed by prepare()'s helix kv_lens branch. - self.helix_owned_new_tokens_cpu = torch.zeros( - (self.max_num_sequences, ), - device='cpu', - dtype=torch.int, - pin_memory=prefer_pinned(), - ) - self._helix_spec_tokens_valid = False + ``ctx_cached_token_indptr`` / ``ctx_kv_indptr`` [max_num_requests + 1] + (int64) and their pinned host copies; filled per step by + ``stage_context_cached_kv`` or the context-MLA preparation. + """ + md = self.metadata + self.ctx_cached_token_indptr = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + cache_name="ctx_cached_token_indptr", + dtype=torch.int64, + capture_graph=capture_graph, + ) + self.host_ctx_cached_token_indptr = torch.zeros_like( + self.ctx_cached_token_indptr, + device='cpu', + pin_memory=prefer_pinned(), + ) + # context full seqlens include cached tokens and uncached tokens + self.ctx_kv_indptr = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + cache_name="ctx_kv_indptr", + dtype=torch.int64, + capture_graph=capture_graph, + ) + self.host_ctx_kv_indptr = torch.zeros_like( + self.ctx_kv_indptr, + device='cpu', + pin_memory=prefer_pinned(), + ) - def on_update_kv_lens(self): - # KV lengths can change between speculative decoding sub-steps. - self.context_plan = None - if self.enable_flash_mla: - self._flash_mla_metadata_valid = False + def adopt_target_state(self, source: "TrtllmCoreMetadata") -> None: + """Take the target's prepared state at a draft-loop start. + + The runtime KV lengths are the draft's own device lengths; its MLA + prefixes and split plan are its own, built lazily from its own lengths. + """ + super().adopt_target_state(source) self._invalidate_mla_scheduler_buffers() - state = getattr(self, "fp4_mla_state", None) - if state is not None: - state.on_update_kv_lens(self) - view = self.mixed_precision_staging - if view is not None and view.manager is self.kv_cache_manager: - # Keep the staging lengths in step with kv_lens_cuda. - lengths = view.lengths - n = lengths.shape[1] - lengths[1].copy_(self.kv_lens_cuda[:n]) - torch.sub(lengths[1], self.seq_lens_kv_cuda[:n], out=lengths[0]) - - def update_for_spec_dec(self) -> None: + num_seqs = self.num_seqs + self.kv_lens_cuda_runtime = self.kv_lens_cuda[:num_seqs] + if getattr(source, "host_request_types_runtime", None) is not None: + self.host_request_types_runtime = self.host_request_types[:num_seqs] + + def derive(self) -> None: + md = self.metadata self.context_plan = None - if self.enable_flash_mla: - self._flash_mla_metadata_valid = False + # Recomputed on first use this iteration; see mla_prepare_scheduler_buffers. self._invalidate_mla_scheduler_buffers() - state = getattr(self, "fp4_mla_state", None) - if state is not None: - state.update_for_spec_dec(self) + self._steady_block_table_key = None + self.steady_decode_kv_lens = None + self._prepared_full_block_offsets = False + extra_attrs = get_model_extra_attrs() + # If model extra attrs is set, attention_metadata is setup in executor. + if extra_attrs is None: + get_global_attrs().attention_metadata = weakref.ref(md) + if self.kv_cache_manager is None: + # Convert the attention metadata to a TRT-LLM no cache attention metadata. + assert self.kv_cache_manager is None, "no cache attention should not have KV cache manager" + assert self._max_seq_len_storage is not None, "max_seq_len should be set for no cache attention" - def _invalidate_mla_scheduler_buffers(self) -> None: - # Spec-dec rewrites q_lens and kv_lens between sub-steps, so the cumulative - # buffers below must be rebuilt before the next MLA layer reads them. - self._mla_scheduler_buffers_valid = False - self._mla_ctx_cu_seqlens_valid = False - self._cute_dsl_mla_split_plan_key = None + # setting kv cache params + self.kv_cache_params = KVCacheParams(use_cache=False, ) - def restore_from_spec_dec(self) -> None: - super().restore_from_spec_dec() - self.context_plan = None - if self.fp4_mla_state is not None: - self.fp4_mla_state.restore_from_spec_dec(self) + # trtllm attn metadata prepare() requires this + self.prompt_lens = self.context_lens - def update_helix_param( - self, - helix_position_offsets: List[int], - helix_is_inactive_rank: List[bool], - helix_owned_new_tokens: Optional[List[int]] = None, - ) -> None: + self.kv_cache_block_offsets = None + self.block_ids_per_seq = None + + prompt_lens = torch.as_tensor( + self.prompt_lens, + dtype=torch.int, + device='cpu', + ) + self.prompt_lens_cpu[:self.num_seqs].copy_(prompt_lens) + copy_h2d_pool_shared(self.prompt_lens_cuda[:self.num_seqs], + self.prompt_lens_cpu[:self.num_seqs], + pack=True) + + # number of tokens in the kv cache for each sequence in the batch + cached_token_lens = torch.tensor( + self.kv_cache_params.num_cached_tokens_per_seq, + dtype=torch.int, + device='cpu', + ) if self.kv_cache_params.use_cache else None + + self._prepare_helix_context_slots(cached_token_lens) + + # number of tokens needed in the kv cache for each sequence after the next pass. + kv_lens = cached_token_lens + \ + self.seq_lens_kv if cached_token_lens is not None else self.seq_lens_kv + if kv_lens is self.seq_lens or kv_lens is self.seq_lens_kv: + kv_lens = kv_lens.clone() + + # self.kv_lens is the valid kv cache length, while the self.kv_lens_cuda is + # the sequence length including the cached tokens and the input tokens. + self.kv_lens[:self.num_seqs].copy_( + kv_lens + self.kv_cache_params.num_extra_kv_tokens) + if self.kv_lens_offsets is not None: + self.kv_lens_cuda[:self.num_seqs].copy_(maybe_pin_memory( + kv_lens[:self.num_seqs]), + non_blocking=True) + # Corrections known only on device; the host copies keep the + # scheduled lengths. + self.kv_lens_cuda[:self.num_seqs] += self.kv_lens_offsets[:self. + num_seqs] + else: + # Nothing on the device touches the lengths before the forward, so + # the copy may join the step-input pack. + step_inputs.copy_(self.kv_lens_cuda[:self.num_seqs], + maybe_pin_memory(kv_lens[:self.num_seqs])) + # total kv lens for context requests and generation requests, without extra tokens + self.host_total_kv_lens[0] = kv_lens[:self.num_contexts].sum().item() + self.host_total_kv_lens[1] = kv_lens[self.num_contexts:self. + num_seqs].sum().item() + self.host_request_types[:self.num_contexts].fill_(0) + self.host_request_types[self.num_contexts:self.num_seqs].fill_(1) + + # prepare for kv cache reuse/chunked context in MLA + if self.enable_context_mla_with_cached_kv: + self.prepare_context_mla_with_cached_kv(cached_token_lens, kv_lens) + + # kv block offsets + assert self.request_ids is not None + if self.kv_cache_manager is not None: + max_kv_len = int(self.kv_lens[:self.num_seqs].max()) + assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( + f"The max KV cache length of input sequences ({max_kv_len}) " + f"exceeds the KV cache manager's maximum supported length " + f"({self.kv_cache_manager.max_seq_len}).") + + # On the non-speculative path the host kv_lens snapshot bounds + # every block-table access, so the staged/H2D width can be capped + # at the batch's maximum instead of max_seq_len's worth of + # columns. Speculative decoding must stage the full width: + # draft/tree sub-steps and the overlap scheduler advance + # kv_lens_cuda on device past the host snapshot, and their + # kernels dereference block columns a host-derived cap would + # leave unstaged (uninitialized in this buffer). + spec_active = (md.is_spec_decoding_enabled + or bool(self.kv_cache_params.num_extra_kv_tokens) or + (self.runtime_features is not None and + self.runtime_features.has_speculative_draft_tokens) + or self.full_block_table_width) + max_blocks = None + if not spec_active and self.kv_cache_manager.tokens_per_block: + max_blocks = ceil_div(max_kv_len, + self.kv_cache_manager.tokens_per_block) + self.kv_cache_manager.prepare_batch_tables(self.request_ids, + self.num_contexts) + self.kv_cache_manager.copy_batch_block_offsets( + self.kv_cache_block_offsets, + self.request_ids, + self.beam_width, + self.num_contexts, + self.num_seqs, + max_blocks=max_blocks) + self._prepared_full_block_offsets = max_blocks is None + + # Don't pass self.kv_lens as kv_lens here because it includes extra + # tokens. Use the actual KV length (without extra tokens) for + # kv_lens_runtime, which becomes host_past_key_value_lengths and + # eventually mMaxSeqLenKv. + self.bind_runtime_views( + kv_lens_cuda=self.kv_lens_cuda[:self.num_seqs], + kv_lens=kv_lens[:self.num_seqs], + prompt_lens_cuda=self.prompt_lens_cuda[:self.num_seqs], + prompt_lens_cpu=self.prompt_lens_cpu[:self.num_seqs], + host_request_types=self.host_request_types[:self.num_seqs], + ) + + if self._has_mixed_precision_staging(): + # Native BF16 paged-context FMHA reads 32-token pages. + self.prepare_mixed_precision_kv( + "HND", min(self.kv_cache_manager.tokens_per_block, 32)) + if self.kv_lens_offsets is not None: + self._sync_mixed_precision_staging_lengths() + + @property + def effective_beam_width(self) -> int: + # Only use this for the fallback kernel's beam_width argument. + # Cross-attention reads request-scoped encoder K/V that is written once + # and reused unchanged by every decoder beam. Metadata preparation still + # uses beam_width to expand cross block-offset rows to decoder-sequence + # scope, but the fallback kernel should treat the cross K/V cache as + # non-beam-packed. + return 1 if self.is_cross else self.beam_width + + @property + def effective_workspace(self) -> Optional[torch.Tensor]: + """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" + return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace + + @property + def max_context_length(self) -> int: + """ + Upper bound for a single context window. + Required max_seq_len for context-only attention cases like visual gen """ - Update helix parameters by copying into static buffers for CUDA graph compatibility. + return min(self.max_seq_len, self.max_num_tokens) - Args: - helix_position_offsets: Position offsets for helix parallelism with shape (num_tokens,). - helix_is_inactive_rank: Whether the current rank is inactive with shape (batch_size,). - helix_owned_new_tokens: Per-sequence count of this step's new - tokens owned by this rank (speculative verify groups; one - group may straddle a page boundary onto two ranks). None on - the single-token path, where the boolean flag carries it. + @property + def max_seq_len(self) -> int: """ - if helix_position_offsets is not None and self.helix_position_offsets is not None: - num_tokens = len(helix_position_offsets) - self.helix_position_offsets_cpu[:num_tokens].copy_( - torch.tensor(helix_position_offsets, dtype=torch.int)) - self.helix_position_offsets[:num_tokens].copy_( - self.helix_position_offsets_cpu[:num_tokens], non_blocking=True) - - if helix_is_inactive_rank is not None and self.helix_is_inactive_rank is not None: - batch_size = len(helix_is_inactive_rank) - self.helix_is_inactive_rank_cpu[:batch_size].copy_( - torch.tensor(helix_is_inactive_rank, dtype=torch.bool)) - self.helix_is_inactive_rank[:batch_size].copy_( - self.helix_is_inactive_rank_cpu[:batch_size], non_blocking=True) - - self._helix_spec_tokens_valid = False - if helix_owned_new_tokens is not None: - batch_size = len(helix_owned_new_tokens) - self.helix_owned_new_tokens_cpu[:batch_size].copy_( - torch.tensor(helix_owned_new_tokens, dtype=torch.int)) - self._helix_spec_tokens_valid = True + Returns the max sequence length. + If the attention uses KV cache, it will return max_seq_len from the KV cache manager. + If the attention is no cache, max_seq_len should be set manually by user. + """ + if self.kv_cache_manager is not None: + return self.kv_cache_manager.max_seq_len + else: + assert self._max_seq_len_storage is not None, "max_seq_len should be set for no kv cache attention" + return self._max_seq_len_storage + + # Block tables given to a metadata without a KV cache manager; with one, + # the core allocates or derives them from the manager. + _manager_block_tables = frozenset({ + "kv_cache_block_offsets", + "host_kv_cache_block_offsets", + "block_ids_per_seq", + "kv_block_ids_per_seq", + }) + + def creation_value(self, name: str) -> Any: + # The value set for no-cache attention; a cache's comes from its manager. + if name == "max_seq_len": + return self._max_seq_len_storage + if name in self._manager_block_tables and self.kv_cache_manager is not None: + return None + return super().creation_value(name) + + @max_seq_len.setter + def max_seq_len(self, value: int) -> None: + """ + Set the max sequence length for no cache attention. + """ + self._max_seq_len_storage = value + + @property + def tokens_per_block(self) -> Optional[int]: + """ + Returns the number of tokens per block from the KV cache manager. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[2] + return self.kv_cache_manager.tokens_per_block if self.kv_cache_manager is not None else None + + @property + def page_size(self) -> int: + """ + Number of tokens per cache page. + """ + page_size_override = getattr(self, '_page_size_override', None) + if page_size_override is not None: + return page_size_override + assert self.kv_cache_manager is not None, "page_size requires a KV cache manager" + return self.kv_cache_manager.tokens_per_block + + @page_size.setter + def page_size(self, value: int) -> None: + self._page_size_override = value + + @property + def host_kv_cache_pool_pointers(self) -> Optional[torch.Tensor]: + """ + Returns the host KV cache pool pointers from the KV cache manager if KV cache manager is not None. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[0] + return self.kv_cache_manager.kv_cache_pool_pointers if self.kv_cache_manager is not None else None + + @property + def host_kv_cache_pool_mapping(self) -> Optional[torch.Tensor]: + """ + Returns the host KV cache pool mapping from the KV cache manager if KV cache manager is not None. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[1] + return self.kv_cache_manager.kv_cache_pool_mapping if self.kv_cache_manager is not None else None + + def _extension_kv_cache_pools( + self) -> Optional[Tuple[torch.Tensor, torch.Tensor, int]]: + # An extension's pools (CSA2 compute tiles) replace the manager's. + return self.metadata.find("kv_cache_pools", None) + + def copy_lengths_from(self, source: "AttentionCoreMetadata") -> None: + super().copy_lengths_from(source) + num_seqs = self.num_seqs + self.kv_lens_cuda[:num_seqs].copy_(source.kv_lens_cuda[:num_seqs]) + if self.host_request_types is not None: + self.host_request_types[:num_seqs].copy_( + source.host_request_types[:num_seqs]) + + def resync_length_derived_state(self) -> None: + # A draft step changed the lengths the layer-invariant prefixes derive + # from; the next MLA layer rebuilds them, and the next context-only + # call plans again. + self.context_plan = None + self._invalidate_mla_scheduler_buffers() + self._sync_mixed_precision_staging_lengths() + + def _sync_mixed_precision_staging_lengths(self) -> None: + """Keep the mixed-precision staging lengths in step with kv_lens_cuda. + + The staging copy loads the host lengths in ``prepare_mixed_precision_kv``; + the step's device corrections (``kv_lens_offsets``) and a draft loop's + length changes reach only kv_lens_cuda, so the staging lengths are + taken from it on the device. + """ + view = self.mixed_precision_staging + if (view is None or view.lengths is None + or view.manager is not self.kv_cache_manager): + return + lengths = view.lengths + n = lengths.shape[1] + lengths[1].copy_(self.kv_lens_cuda[:n]) + torch.sub(lengths[1], self.seq_lens_kv_cuda[:n], out=lengths[0]) + + def _invalidate_mla_scheduler_buffers(self) -> None: + # Spec-dec rewrites q_lens and kv_lens between sub-steps, so the cumulative + # buffers below must be rebuilt before the next MLA layer reads them. + self._mla_scheduler_buffers_valid = False + self._mla_ctx_cu_seqlens_valid = False + self._cute_dsl_mla_split_plan_key = None def _prepare_helix_context_slots( self, cached_token_lens: Optional[torch.Tensor]) -> None: @@ -835,74 +892,7 @@ def _prepare_helix_context_slots( self.helix_context_local_slots = maybe_pin_memory(slots.to( torch.int32)).to(self.seq_lens_cuda.device, non_blocking=True) - def helix_local_len_vec(self, global_lens: torch.Tensor) -> torch.Tensor: - """Vectorized rank-local prefix length for helix round-robin pages. - - For each global sequence length g, returns the number of the first g - tokens whose ledger page lives on this CP rank (page b -> rank - b % cp_size). The rule and its scalar twin live in - ``_torch.utils``; KVCacheManagerV2._helix_local_len and the host - packing in model_engine use the same definition. - """ - return helix_local_len_tensor(global_lens, - self.kv_cache_manager.tokens_per_block, - self.mapping.cp_size, - self.mapping.cp_rank) - - def recompute_helix_spec_buffers(self, num_gen_tokens: int, - tokens_per_gen_seq: int) -> None: - """Derive per-token helix buffers from (corrected) global positions. - - Called after the overlap-scheduler device correction has been applied - to helix_position_offsets, so every derived quantity reflects the - real committed length even though the host packed provisional values. - Static shapes only; safe under CUDA graph capture. - - ``tokens_per_gen_seq`` is the uniform verify-group width. Non-uniform - groups are rejected rather than silently mis-sliced: without the - overlap scheduler the extend loop packs a per-request - ``1 + get_draft_token_length(request)`` and a request entering with no - draft tokens is packed as a single-token generation row instead, so a - batch can arrive whose total happens to divide but whose rows do not - line up. - - Two index bases meet here, and they are not the same: - * helix_position_offsets / helix_local_slots / helix_kv_bounds are - GENERATION-RELATIVE -- the packing loops only append for extend - and generation rows, so token 0 is the first generation token. - * kv_lens_cuda is BATCH-indexed, hence the num_contexts offset on - the write below. - """ - assert num_gen_tokens % tokens_per_gen_seq == 0, ( - f"helix spec expects uniform verify groups: {num_gen_tokens} gen " - f"tokens not divisible by group size {tokens_per_gen_seq}") - num_gen_seqs = num_gen_tokens // tokens_per_gen_seq - # Divisibility alone does not imply uniformity: a batch of mixed group - # widths can still divide and would then write the wrong number of - # kv_lens_cuda rows with values taken from the wrong tokens. - assert num_gen_seqs == self.num_generations, ( - f"helix spec expects uniform verify groups: {num_gen_tokens} gen " - f"tokens over {self.num_generations} generation rows do not all " - f"have width {tokens_per_gen_seq}") - # One launch derives the write slots, the attention bounds and the - # per-sequence rank-local kv length (the bound of the sequence's last - # token: committed + owned in-flight tokens) from the positions; it - # replaces a chain of elementwise ops on the step's critical path. - # Static shapes and allocation-free, so it is safe under CUDA graph - # capture. - populate_helix_spec_buffers( - self.helix_position_offsets[:num_gen_tokens], - self.helix_local_slots[:num_gen_tokens], - self.helix_kv_bounds[:num_gen_tokens], - self.kv_lens_cuda[self.num_contexts:self.num_contexts + - num_gen_seqs], - num_tokens=num_gen_tokens, - tokens_per_seq=tokens_per_gen_seq, - tokens_per_block=self.kv_cache_manager.tokens_per_block, - cp_size=self.mapping.cp_size, - cp_rank=self.mapping.cp_rank) - - def _bind_runtime_views( + def bind_runtime_views( self, *, kv_lens_cuda: torch.Tensor, @@ -923,6 +913,357 @@ def _bind_runtime_views( self.prompt_lens_cpu_runtime: torch.Tensor = prompt_lens_cpu self.host_request_types_runtime = host_request_types + def substitute_runtime_views( + self, + *, + kv_lens_cuda: Optional[torch.Tensor] = None, + kv_lens: Optional[torch.Tensor] = None, + prompt_lens_cuda: Optional[torch.Tensor] = None, + prompt_lens_cpu: Optional[torch.Tensor] = None, + context_total_kv_len: Optional[int] = None, + ) -> None: + """Present other tensors through the given ``*_runtime`` views. + + For callers whose kernels read different lengths through the same + views (FP4 MLA reads the per-forward append lengths; chunked MLA + context attention reads one chunk's). ``context_total_kv_len`` is the + context KV total the op reads with them. Views not given stay bound; + nothing is announced, so it may run inside an event. + """ + if kv_lens_cuda is not None: + self.kv_lens_cuda_runtime = kv_lens_cuda + if kv_lens is not None: + self.kv_lens_runtime = kv_lens + if context_total_kv_len is not None: + self.host_total_kv_lens[0] = context_total_kv_len + if prompt_lens_cuda is not None: + self.prompt_lens_cuda_runtime = prompt_lens_cuda + if prompt_lens_cpu is not None: + self.prompt_lens_cpu_runtime = prompt_lens_cpu + + def republish_kv_lens(self, kv_lens: torch.Tensor) -> None: + """Publish corrected per-sequence KV lengths after ``derive``. + + ``kv_lens`` [num_seqs] (host, without the extra KV tokens) replaces the + derived lengths: the host and device copies, the context/generation + totals and the runtime views. For an extension whose kernels attend to + other lengths than the batch's (Helix's rank-local generation rows), + called from its ``prepare``. + """ + num_seqs = self.num_seqs + kv_lens = kv_lens[:num_seqs] + self.kv_lens[:num_seqs].copy_(kv_lens + + self.kv_cache_params.num_extra_kv_tokens) + self.kv_lens_cuda[:num_seqs].copy_(maybe_pin_memory(kv_lens), + non_blocking=True) + self.host_total_kv_lens[0] = kv_lens[:self.num_contexts].sum().item() + self.host_total_kv_lens[1] = kv_lens[self.num_contexts:].sum().item() + self.bind_runtime_views( + kv_lens_cuda=self.kv_lens_cuda[:num_seqs], + kv_lens=kv_lens, + prompt_lens_cuda=self.prompt_lens_cuda_runtime, + prompt_lens_cpu=self.prompt_lens_cpu_runtime, + host_request_types=self.host_request_types_runtime, + ) + + def override_cached_token_lens( + self, num_cached_tokens_per_seq: List[int]) -> None: + super().override_cached_token_lens(num_cached_tokens_per_seq) + if not self.kv_cache_params.use_cache: + return + cached = torch.tensor(num_cached_tokens_per_seq[:self.num_seqs], + dtype=torch.int, + device='cpu') + self.republish_kv_lens(cached + self.seq_lens_kv[:self.num_seqs]) + + def override_prompt_lens(self, prompt_lens: List[int]) -> None: + """Present ``prompt_lens`` to the kernels in place of the batch's.""" + num_seqs = self.num_seqs + self.prompt_lens_cpu[:num_seqs].copy_( + torch.tensor(prompt_lens, dtype=torch.int, device="cpu")) + self.prompt_lens_cuda[:num_seqs].copy_(self.prompt_lens_cpu[:num_seqs], + non_blocking=True) + self.substitute_runtime_views( + prompt_lens_cuda=self.prompt_lens_cuda[:num_seqs], + prompt_lens_cpu=self.prompt_lens_cpu[:num_seqs]) + + def context_scratch_view( + self, + *, + kv_cache_manager: Any, + kv_cache_block_offsets: torch.Tensor, + block_ids_per_seq: Optional[torch.Tensor], + host_total_kv_lens: torch.Tensor, + kv_lens_cuda: torch.Tensor, + kv_lens: torch.Tensor, + position_offsets: torch.Tensor, + without: Tuple[Type[MetadataExtension], ...] = (), + ) -> "TrtllmAttentionMetadata": + """A view of this metadata over a disposable cache of the context requests. + + For kernels that write and read the context K/V in a scratch cache + (FP4 MLA's FP8 context path): the view addresses ``kv_cache_manager`` + through ``kv_cache_block_offsets``, presents only the context rows + with ``kv_lens_cuda`` / ``kv_lens`` as their KV lengths and gives the + tokens' absolute positions as Helix position offsets. The extensions + in ``without`` are left out. The view shares every buffer with this + metadata and is valid for the current step. + """ + view = copy.copy(self.metadata) + for extension_type in without: + view.remove_ext(extension_type) + core = view.core() + num_contexts = core.num_contexts + core.kv_cache_manager = kv_cache_manager + core.kv_cache_block_offsets = kv_cache_block_offsets + core.block_ids_per_seq = block_ids_per_seq + core.prompt_lens_cuda_runtime = core.prompt_lens_cuda_runtime[: + num_contexts] + core.prompt_lens_cpu_runtime = core.prompt_lens_cpu_runtime[: + num_contexts] + core.host_request_types_runtime = core.host_request_types_runtime[: + num_contexts] + core.kv_lens_cuda_runtime = kv_lens_cuda + core.kv_lens_runtime = kv_lens + core.host_total_kv_lens = host_total_kv_lens + helix = view.find_ext(HelixMetadata) + if helix is None: + # Without Helix the view still presents the positions through + # Helix state; it is built once and its buffers stay unused. + if self._context_scratch_helix is None: + self._context_scratch_helix = HelixMetadata(view) + helix = self._context_scratch_helix + helix = copy.copy(helix) + helix.metadata = view + helix.helix_position_offsets = position_offsets + view.replace_ext(helix) + return view + + def override_page_size(self, page_size: int) -> None: + """Present ``page_size`` to the kernels in place of the manager's.""" + self.page_size = page_size + + def set_num_sparse_topk(self, num_sparse_topk: int) -> None: + """Set the KV count each sparse attention query attends to.""" + self.num_sparse_topk = num_sparse_topk + + def set_use_paged_context_fmha(self, enabled: bool) -> None: + """Choose whether context attention reads K/V from the paged cache.""" + self.use_paged_context_fmha = enabled + + def disable_workspace_reclamation(self) -> None: + """Keep the workspace across forwards (graphs or staged FMHA state).""" + self.workspace_reclaimable = False + + def use_block_id_tables(self, block_ids_per_seq: torch.Tensor, + kv_block_ids_per_seq: torch.Tensor) -> None: + """Bind the per-sequence block-ID tables FlashMLA kernels read.""" + self.block_ids_per_seq = block_ids_per_seq + self.kv_block_ids_per_seq = kv_block_ids_per_seq + + def as_generation_rows(self) -> None: + """Continue every context row as a generation row; see the base method. + + The attention op reads the rows' request types from + ``host_request_types``, and generation rows have no cached context KV, + so both are reset with the context count. + """ + # Allocated with the core's buffers; a core built without them has none. + host_request_types = getattr(self, "host_request_types", None) + if host_request_types is not None: + host_request_types[:self.num_contexts].fill_(1) + super().as_generation_rows() + self.clear_context_cached_kv() + + def clear_context_cached_kv(self) -> None: + """Mark the batch as having no cached context KV (generation sub-steps).""" + self.max_ctx_kv_len = 0 + self.num_ctx_cached_tokens = 0 + + def stage_context_cached_kv(self, cached_token_lens: torch.Tensor, + kv_lens: torch.Tensor) -> None: + """Publish the context requests' cached-token and KV-length prefixes. + + Fills ``ctx_cached_token_indptr`` / ``ctx_kv_indptr`` (see + ``allocate_context_kv_indptrs``) and the context maxima from the + per-sequence host lengths. + """ + num_contexts = self.num_contexts + if num_contexts > 0: + self.num_ctx_cached_tokens = cached_token_lens[:num_contexts].sum( + ).item() + self.max_ctx_kv_len = kv_lens[:num_contexts].max().item() + self.max_ctx_seq_len = self.seq_lens[:num_contexts].max().item() + torch.cumsum( + cached_token_lens[:num_contexts], + dim=0, + dtype=torch.int64, + out=self.host_ctx_cached_token_indptr[1:num_contexts + 1], + ) + self.ctx_cached_token_indptr[:num_contexts + 1].copy_( + self.host_ctx_cached_token_indptr[:num_contexts + 1], + non_blocking=True) + torch.cumsum( + kv_lens[:num_contexts], + dim=0, + dtype=torch.int64, + out=self.host_ctx_kv_indptr[1:num_contexts + 1], + ) + self.ctx_kv_indptr[:num_contexts + 1].copy_( + self.host_ctx_kv_indptr[:num_contexts + 1], non_blocking=True) + else: + self.num_ctx_cached_tokens = 0 + self.max_ctx_kv_len = 0 + self.max_ctx_seq_len = 0 + + def bind_fixed_kv_window(self, kv_len: int, + kv_cache_block_offsets: torch.Tensor) -> None: + """Address a fixed staging window instead of the paged cache. + + Every sequence is a generation request attending to ``kv_len`` keys + through ``kv_cache_block_offsets`` (CSA2 query tiles); the workspace + is the one graph captures use. + """ + self.num_sparse_topk = kv_len + self.max_seq_len = kv_len + self.kv_cache_params = KVCacheParams(use_cache=True) + self.kv_cache_block_offsets = kv_cache_block_offsets + self.kv_lens.fill_(kv_len) + self.kv_lens_cuda.fill_(kv_len) + self.prompt_lens_cpu.fill_(1) + self.prompt_lens_cuda.fill_(1) + self.host_request_types.fill_(1) + self.host_total_kv_lens.zero_() + # Each query count has independent native workspace storage. + # Eager initialization and capture must use that same tensor. + self.cuda_graph_workspace = self.workspace + + def use_block_offsets(self, kv_cache_block_offsets: torch.Tensor) -> None: + """Address the paged KV cache through ``kv_cache_block_offsets`` after ``derive``. + + For a KV cache whose page lists the attention op reads from the + cache's own tensor (VisualGen's block-causal cache) instead of the + block offsets ``derive`` copied. + """ + self.kv_cache_block_offsets = kv_cache_block_offsets + + def use_workspace(self, workspace: torch.Tensor) -> None: + """Run on ``workspace`` both eagerly and in graph capture.""" + self.workspace = self.cuda_graph_workspace = workspace + + def reallocate_workspace(self, num_bytes: int) -> None: + """Replace the eager workspace with a new one of ``num_bytes`` on its device. + + The old storage is released before the new one is allocated: + shrinking it in place would retain it. + """ + device = self.workspace.device + self.workspace = None + self.workspace = torch.empty(num_bytes, dtype=torch.int8, device=device) + + def mark_trtllm_gen_jit_warmup(self, enabled: bool) -> None: + """Mark whether the next forwards are the TRTLLM-Gen FMHA JIT warmup.""" + self.trtllm_gen_jit_warmup = enabled + + def bind_generation_tile(self, query_lens_host: torch.Tensor, + query_lens_device: torch.Tensor, + q_offsets: torch.Tensor, + kv_offsets: torch.Tensor) -> None: + """Bind a fixed-window generation batch over the caller's buffers. + + One single-token request per query row (see ``bind_fixed_kv_window``). + """ + count = query_lens_host.numel() + self.bind_lengths(query_lens_host, + query_lens_device, + num_contexts=0, + num_ctx_tokens=0, + num_generations=count, + num_tokens=count) + self.bind_runtime_views( + kv_lens_cuda=self.kv_lens_cuda, + kv_lens=self.kv_lens, + prompt_lens_cuda=self.prompt_lens_cuda, + prompt_lens_cpu=self.prompt_lens_cpu, + host_request_types=self.host_request_types, + ) + self.host_total_kv_lens[1] = count * self.num_sparse_topk + self.cu_q_seqlens = q_offsets + self.cu_kv_seqlens = kv_offsets + + def bind_context_tile( + self, + query_lens: torch.Tensor, + kv_lens: torch.Tensor, + *, + query_lens_host: torch.Tensor, + query_lens_device: torch.Tensor, + q_offsets: torch.Tensor, + kv_offsets: torch.Tensor, + copy_to_device: Callable[[str, torch.Tensor, torch.Tensor], None], + ) -> None: + """Bind a fixed-window context batch over the caller's buffers. + + Request ``i`` has ``query_lens[i]`` query tokens attending to + ``kv_lens[i]`` keys (host int32 tensors); ``query_lens_host`` / + ``query_lens_device`` already hold the query lengths. + ``copy_to_device(key, destination, source)`` stages host data. + """ + requests = query_lens.numel() + num_tokens = int(query_lens.sum()) + self.bind_lengths(query_lens_host, + query_lens_device, + num_contexts=requests, + num_ctx_tokens=num_tokens, + num_generations=0, + num_tokens=num_tokens) + self.kv_lens[:requests].copy_(kv_lens) + copy_to_device("context_kv_lens", self.kv_lens_cuda[:requests], kv_lens) + # THOP interprets context_lengths as Q lengths, independently of + # past/current KV lengths. Keep them consistent with unfolded cuQ. + self.prompt_lens_cpu[:requests].copy_(query_lens) + copy_to_device("context_prompt_lens", self.prompt_lens_cuda[:requests], + query_lens) + self.host_request_types[:requests].zero_() + self.host_total_kv_lens.zero_() + self.host_total_kv_lens[0] = int(kv_lens.sum()) + self.max_seq_len = max(self.num_sparse_topk, int(kv_lens.max())) + self.cu_q_seqlens = q_offsets + self.cu_kv_seqlens = kv_offsets + self.bind_runtime_views( + kv_lens_cuda=self.kv_lens_cuda[:requests], + kv_lens=self.kv_lens[:requests], + prompt_lens_cuda=self.prompt_lens_cuda[:requests], + prompt_lens_cpu=self.prompt_lens_cpu[:requests], + host_request_types=self.host_request_types[:requests], + ) + + def cute_dsl_mla_split_plan( + self, + batch_size: int, + device: torch.device, + key: tuple, + launch: Callable[[torch.Tensor], None], + ) -> Optional[torch.Tensor]: + """This step's split-KV plan of the CuTe DSL MLA decode, ``[batch_size]``. + + ``launch(plan)`` fills the plan buffer; it runs once per step, again + only when ``key`` or the buffer changes, or after the lengths changed. + None when the buffer would have to be allocated during graph capture. + """ + plan = self._cute_dsl_mla_split_plan + if plan is None or plan.numel() < batch_size or plan.device != device: + if torch.cuda.is_current_stream_capturing(): + return None + capacity = max(int(self.max_num_requests or 0), batch_size) + plan = torch.empty(capacity, dtype=torch.int32, device=device) + self._cute_dsl_mla_split_plan = plan + full_key = (*key, plan.data_ptr()) + if self._cute_dsl_mla_split_plan_key != full_key: + launch(plan) + self._cute_dsl_mla_split_plan_key = full_key + return plan[:batch_size] + def mla_prepare_scheduler_buffers( self, num_heads: int) -> tuple[torch.Tensor, torch.Tensor]: """Cumulative Q rows / KV lengths for the MLA generation FMHA kernel. @@ -940,6 +1281,11 @@ def mla_prepare_scheduler_buffers( touches the device copy. Results land in fixed-address buffers so a captured CUDA graph keeps reading the same pointers, and no D2H sync is introduced. """ + md = self.metadata + extension_buffers = md.find("mla_scheduler_buffers", None) + buffers = extension_buffers(num_heads) if extension_buffers else None + if buffers is not None: + return buffers num_ctx = self.num_contexts n_gen = self.num_seqs - num_ctx if not self._mla_scheduler_buffers_valid: @@ -977,47 +1323,10 @@ def mla_prepare_ctx_cu_seqlens(self) -> Optional[torch.Tensor]: self._mla_ctx_cu_seqlens_valid = True return self.mla_ctx_cu_q_seqlens[:num_ctx + 1] - def record_draft_swap(self, name: str) -> None: - """Record name's current value so restore_draft_swaps can rebind it.""" - self.draft_replay_swapped_attrs.setdefault(name, - getattr(self, name, None)) - - def swap_for_draft(self, name: str, draft_value: Any) -> None: - """Rebind name to draft_value, recording the original first.""" - self.record_draft_swap(name) - setattr(self, name, draft_value) - self.context_plan = None - - def restore_draft_swaps(self) -> None: - """Rebind every recorded attribute to its original and clear the record. - - Swaps do not nest: this restores all swaps made since the last restore. - """ - for name, original in self.draft_replay_swapped_attrs.items(): - setattr(self, name, original) - self.draft_replay_swapped_attrs = {} - self.context_plan = None - - def prepare_for_draft_forward(self) -> dict | None: - """Prepare backend state shared by draft-forward execution paths.""" - draft = self.sync_draft_mixed_precision_staging() - if draft is not None: - # MTP layers of a mixed-precision draft manager run on its own - # staging copy. - self.swap_for_draft("mixed_precision_staging", draft) - return None - - def restore_after_draft_forward(self, saved_state: dict | None) -> None: - """Restore backend state modified for draft-forward execution.""" - return None - - def token_major_gen_view(self) -> Optional["TokenMajorGenView"]: - """The token-major generation presentation, or None. - - None on every backend but DSA, and on every uniform DSA batch. See - ``DSAtrtllmAttentionMetadata.token_major_gen_view``. - """ - return None + def token_major_view(self) -> Optional["TokenMajorGenView"]: + """The one-row-per-query-token generation view an extension provides, or None.""" + view = self.metadata.find("token_major_gen_view", None) + return view() if view is not None else None @contextlib.contextmanager def presented_token_major( @@ -1070,184 +1379,6 @@ def presented_token_major( self.host_request_types_runtime, self._num_generations, self.max_num_sequences) = saved - def prepare(self) -> None: - super().prepare() - self.context_plan = None - # Recomputed on first use this iteration; see mla_prepare_scheduler_buffers. - self._invalidate_mla_scheduler_buffers() - self._steady_block_table_keys = (None, None) - self._prepared_full_block_offsets = False - extra_attrs = get_model_extra_attrs() - # If model extra attrs is set, attention_metadata is setup in executor. - if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(self) - if self.kv_cache_manager is None: - # Convert the attention metadata to a TRT-LLM no cache attention metadata. - assert self.kv_cache_manager is None, "no cache attention should not have KV cache manager" - assert self._max_seq_len_storage is not None, "max_seq_len should be set for no cache attention" - - # setting kv cache params - self.kv_cache_params = KVCacheParams(use_cache=False, ) - - # trtllm attn metadata prepare() requires this - self.prompt_lens = self.context_lens - - self.kv_cache_block_offsets = None - self.block_ids_per_seq = None - - prompt_lens = torch.as_tensor( - self.prompt_lens, - dtype=torch.int, - device='cpu', - ) - self.prompt_lens_cpu[:self.num_seqs].copy_(prompt_lens) - copy_h2d_pool_shared(self.prompt_lens_cuda[:self.num_seqs], - self.prompt_lens_cpu[:self.num_seqs], - pack=True) - - # number of tokens in the kv cache for each sequence in the batch - cached_token_lens = torch.tensor( - self.kv_cache_params.num_cached_tokens_per_seq, - dtype=torch.int, - device='cpu', - ) if self.kv_cache_params.use_cache else None - - self._prepare_helix_context_slots(cached_token_lens) - - if self.enable_flash_mla: - self.prepare_flash_mla() - - # number of tokens needed in the kv cache for each sequence after the next pass. - if self.enable_helix: - # If helix is inactive, attend to the previously cached tokens only. - assert cached_token_lens is not None, "cached_token_lens should be set for helix" - # The helix per-sequence buffers are GENERATION-relative: the - # packing loops in model_engine append only for extend and plain - # generation rows, so update_helix_param writes exactly - # [0, num_generations). Every device consumer indexes them the - # same way (the MLA rope generation kernel, the XQA preprocessing - # kernels, the FP4 MLA generation kernel), so the host read cannot - # slice them from 0 against a contexts-first cached_token_lens: - # that both shifts every pairing by num_contexts and reads past - # the written region, which for helix_is_inactive_rank_cpu is - # uninitialized memory. Pair them with the generation slice of the - # batch-indexed tensors instead. - num_gen = self.num_generations - gen = slice(self.num_contexts, self.num_seqs) - # Context rows are not part of a verify group and are not packed - # into the helix buffers at all; they append every one of their - # tokens, exactly like the non-helix path below. - kv_lens = cached_token_lens + self.seq_lens_kv - if self._helix_spec_tokens_valid: - # Speculative verify groups: a group may straddle a page - # boundary, so ownership of this step's new tokens is a - # per-sequence COUNT, not a boolean. Provisional host values; - # recompute_helix_spec_buffers overrides the device copy - # after the overlap correction. - kv_lens[gen] = (cached_token_lens[gen] + - self.helix_owned_new_tokens_cpu[:num_gen]) - else: - inactive_rank = self.helix_is_inactive_rank_cpu[:num_gen] - kv_lens[gen] = torch.where(inactive_rank, - cached_token_lens[gen], kv_lens[gen]) - else: - kv_lens = cached_token_lens + \ - self.seq_lens_kv if cached_token_lens is not None else self.seq_lens_kv - - # self.kv_lens is the valid kv cache length, while the self.kv_lens_cuda is - # the sequence length including the cached tokens and the input tokens. - self.kv_lens[:self.num_seqs].copy_( - kv_lens + self.kv_cache_params.num_extra_kv_tokens) - step_inputs.copy_(self.kv_lens_cuda[:self.num_seqs], - maybe_pin_memory(kv_lens[:self.num_seqs])) - # total kv lens for context requests and generation requests, without extra tokens - self.host_total_kv_lens[0] = kv_lens[:self.num_contexts].sum().item() - self.host_total_kv_lens[1] = kv_lens[self.num_contexts:self. - num_seqs].sum().item() - self.host_request_types[:self.num_contexts].fill_(0) - self.host_request_types[self.num_contexts:self.num_seqs].fill_(1) - - # prepare for kv cache reuse/chunked context in MLA - if self.enable_context_mla_with_cached_kv: - self.prepare_context_mla_with_cached_kv(cached_token_lens, kv_lens) - - # kv block offsets - assert self.request_ids is not None - if self.kv_cache_manager is not None: - max_kv_len = int(self.kv_lens[:self.num_seqs].max()) - assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( - f"The max KV cache length of input sequences ({max_kv_len}) " - f"exceeds the KV cache manager's maximum supported length " - f"({self.kv_cache_manager.max_seq_len}).") - - # On the non-speculative path the host kv_lens snapshot bounds - # every block-table access, so the staged/H2D width can be capped - # at the batch's maximum instead of max_seq_len's worth of - # columns. Speculative decoding must stage the full width: - # draft/tree sub-steps and the overlap scheduler advance - # kv_lens_cuda on device past the host snapshot, and their - # kernels dereference block columns a host-derived cap would - # leave unstaged (uninitialized in this buffer). - spec_active = (self.draft_kv_cache_manager is not None - or self.is_spec_decoding_enabled - or bool(self.kv_cache_params.num_extra_kv_tokens) or - (self.runtime_features is not None and - self.runtime_features.has_speculative_draft_tokens)) - max_blocks = None - if not spec_active and self.kv_cache_manager.tokens_per_block: - max_blocks = ceil_div(max_kv_len, - self.kv_cache_manager.tokens_per_block) - self.kv_cache_manager.copy_batch_block_offsets( - self.kv_cache_block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - self.num_seqs, - max_blocks=max_blocks) - - # Also prepare draft KV cache block offsets if draft_kv_cache_manager exists - if self.draft_kv_cache_manager is not None: - # Use the wrapper method which works for both V1 and V2 - self.draft_kv_cache_manager.copy_batch_block_offsets( - self.draft_kv_cache_block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - self.num_seqs, - max_blocks=max_blocks) - self._prepared_full_block_offsets = max_blocks is None - - # Don't pass self.kv_lens as kv_lens here because it includes extra - # tokens. Use the actual KV length (without extra tokens) for - # kv_lens_runtime, which becomes host_past_key_value_lengths and - # eventually mMaxSeqLenKv. - if self.fp4_mla_state is not None: - # FP4 MLA needs the per-forward append lengths rather than the - # original prompt lengths. Context scratch-cache metadata and MTP - # generation both consume these runtime views. - kv_lens_runtime = kv_lens[:self.num_seqs] - prompt_lens_cuda_runtime = self.seq_lens_kv_cuda[:self.num_seqs] - prompt_lens_cpu_runtime = self.seq_lens_kv[:self.num_seqs] - else: - kv_lens_runtime = kv_lens[:self.num_seqs] - prompt_lens_cuda_runtime = self.prompt_lens_cuda[:self.num_seqs] - prompt_lens_cpu_runtime = self.prompt_lens_cpu[:self.num_seqs] - self._bind_runtime_views( - kv_lens_cuda=self.kv_lens_cuda[:self.num_seqs], - kv_lens=kv_lens_runtime, - prompt_lens_cuda=prompt_lens_cuda_runtime, - prompt_lens_cpu=prompt_lens_cpu_runtime, - host_request_types=self.host_request_types[:self.num_seqs], - ) - - if self.fp4_mla_state is not None: - self.fp4_mla_state.prepare(self, kv_lens) - - if self._has_mixed_precision_staging(): - # Native BF16 paged-context FMHA reads 32-token pages. - self.prepare_mixed_precision_kv( - "HND", min(self.kv_cache_manager.tokens_per_block, 32)) - def _on_mixed_precision_kv_prepared(self, view) -> None: """Lay out the staging pages as TRTLLM block offsets for this batch.""" batch = len(view.page_counts) @@ -1287,21 +1418,34 @@ def _restore_mixed_precision_kv(self, saved: tuple) -> None: self.kv_cache_block_offsets = saved[2] def prepare_encoder_decoder_from_precomputed_lengths( - self, prompt_lens: torch.Tensor, kv_lens: torch.Tensor, + self, inputs: BatchInputs, *, num_ctx_tokens: int, + num_generations: int, num_tokens: int, kv_lens: torch.Tensor, context_kv_tokens: int, generation_kv_tokens: int, max_kv_len: int) -> None: - """Prepare encoder-decoder attention from precomputed lengths.""" - super().prepare() + """Prepare encoder-decoder attention from precomputed lengths. + + ``inputs.seq_lens`` are bound as given with the precomputed token and + request counts instead of reductions over them; ``inputs.prompt_lens`` + is the host prompt-length tensor. + """ + md = self.metadata + self.apply_batch_inputs(dataclasses.replace(inputs, seq_lens=None)) + self.bind_lengths(inputs.seq_lens, + num_contexts=inputs.num_contexts, + num_ctx_tokens=num_ctx_tokens, + num_generations=num_generations, + num_tokens=num_tokens) + prompt_lens = inputs.prompt_lens extra_attrs = get_model_extra_attrs() if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(self) + get_global_attrs().attention_metadata = weakref.ref(self.metadata) assert self.kv_cache_manager is not None - assert self.draft_kv_cache_manager is None - assert not self.is_spec_decoding_enabled + assert md.find("draft_or_none", None) is None + assert not md.is_spec_decoding_enabled assert self.kv_cache_params.num_extra_kv_tokens == 0 - assert not self.enable_flash_mla - assert not self.enable_helix + assert md.find_ext(FlashMlaMetadata) is None + assert md.find_ext(HelixMetadata) is None assert not self.enable_context_mla_with_cached_kv assert self.request_ids is not None assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( @@ -1328,7 +1472,7 @@ def prepare_encoder_decoder_from_precomputed_lengths( self.num_contexts, num_seqs, max_blocks=max_blocks) - self._bind_runtime_views( + self.bind_runtime_views( kv_lens_cuda=self.kv_lens_cuda[:num_seqs], kv_lens=kv_lens, prompt_lens_cuda=self.prompt_lens_cuda[:num_seqs], @@ -1338,46 +1482,75 @@ def prepare_encoder_decoder_from_precomputed_lengths( @property def supports_steady_decode_step(self) -> bool: - """Whether prepare_steady_decode_step may stand in for this metadata's prepare(): no host state of - prepare() beyond the KV lengths and block offsets (no FlashMLA, helix, cached-KV context MLA, FP4 MLA, - mixed-precision KV or full generation page table) and prepare() is this class's.""" - return (type(self).prepare is TrtllmAttentionMetadata.prepare - and self.kv_cache_manager is not None - and not self.enable_flash_mla and not self.enable_helix + """Whether prepare_steady_decode_step may stand in for this metadata's prepare. + + Only when prepare derives nothing beyond the KV lengths and block + offsets: no extension but spec-dec mode and the one-model draft (no + FlashMLA, Helix, FP4 MLA, sparse, recurrent or cross-attention state), + no cached-KV context MLA, mixed-precision KV or full generation page + table, and a draft metadata that supports it too (its own steady step + follows the target's). + """ + md = self.metadata + for extension in md._attached_extensions(): + if extension is not self and not isinstance( + extension, (SpecDecodingMetadata, DraftMetadata)): + return False + draft = md.find("draft_or_none", None) + if draft is not None and not draft.supports_steady_decode_step: + return False + # A draft metadata takes its KV-cache parameters from the target when + # it first refreshes; until then the target's decide. + kv_cache_params = self.kv_cache_params + return (self.kv_cache_manager is not None and not self.enable_context_mla_with_cached_kv - and self.fp4_mla_state is None and not self._has_mixed_precision_staging() - and not self.kv_cache_params.use_full_generation_page_table) + and (kv_cache_params is None + or not kv_cache_params.use_full_generation_page_table)) def key_prepared_block_offsets(self) -> None: - """Key the block offsets the last prepare() copied, so that the next prepare_steady_decode_step skips - its copy while they are unchanged. A no-op unless that prepare() copied the full table width.""" + """Key the block offsets the last prepare copied, so that the next + prepare_steady_decode_step skips its copy while they are unchanged. + + A no-op unless that prepare copied the full table width. + """ if not self._prepared_full_block_offsets: return - keys = [] - for manager in (self.kv_cache_manager, self.draft_kv_cache_manager): - keys.append(None if manager is None else manager. - batch_block_table_key(self.request_ids)) - self._steady_block_table_keys = tuple(keys) - - def prepare_steady_decode_step(self, kv_lens: List[int]) -> None: - """prepare() of a generation-only step that runs the batch of the previous prepare() one step later. - - Requires supports_steady_decode_step and the same requests, prompt lengths, request types and beam width - as that prepare(); kv_lens_cuda[:num_seqs] already holds the new lengths on the device. Sets the host - state that follows from ``kv_lens`` (KV lengths without the extra speculative tokens) and copies the - block offsets of a KV cache manager only when its batch_block_table_key changed. Speculative decoding - copies the full table width (max_blocks None), as prepare() does. + self._steady_block_table_key = self.kv_cache_manager.batch_block_table_key( + self.request_ids) + + def prepare_steady_decode_step(self, inputs: BatchInputs, + kv_lens: List[int]) -> None: + """The prepare of a generation-only step that runs the batch of the previous prepare one step later. + + Requires supports_steady_decode_step and the same requests, prompt + lengths, request types and beam width as that prepare. ``inputs`` is + the step's batch description and ``kv_lens`` its scheduled KV lengths + (without the extra speculative tokens). kv_lens_cuda[:num_seqs] + already holds the scheduled lengths on the device, written by the + engine's staging kernel; ``inputs.kv_lens_offsets`` (the overlap + scheduler's corrections) is added to them here, as prepare does. + Sets the host state that follows from ``kv_lens``, copies the block + offsets of a KV cache manager only when its batch_block_table_key + changed (the full table width, as speculative decoding's prepare + does), then prepares the other extensions. """ - AttentionMetadata.prepare(self) + md = self.metadata + self.apply_batch_inputs(inputs) + self.steady_decode_kv_lens = list(kv_lens) + self.context_plan = None self._invalidate_mla_scheduler_buffers() if get_model_extra_attrs() is None: - get_global_attrs().attention_metadata = weakref.ref(self) + get_global_attrs().attention_metadata = weakref.ref(md) num_seqs = self.num_seqs assert len(kv_lens) == num_seqs and self.num_contexts == 0 num_extra_kv_tokens = self.kv_cache_params.num_extra_kv_tokens host_kv_lens = torch.tensor(kv_lens, dtype=torch.int) self.kv_lens[:num_seqs].copy_(host_kv_lens + num_extra_kv_tokens) + if self.kv_lens_offsets is not None: + # Corrections known only on device; the host copies keep the + # scheduled lengths. + self.kv_lens_cuda[:num_seqs] += self.kv_lens_offsets[:num_seqs] self.host_total_kv_lens[0] = 0 self.host_total_kv_lens[1] = sum(kv_lens) max_kv_len = max(kv_lens) + num_extra_kv_tokens @@ -1385,40 +1558,67 @@ def prepare_steady_decode_step(self, kv_lens: List[int]) -> None: f"The max KV cache length of input sequences ({max_kv_len}) " f"exceeds the KV cache manager's maximum supported length " f"({self.kv_cache_manager.max_seq_len}).") - keys = [] - for manager, block_offsets, last_key in zip( - (self.kv_cache_manager, self.draft_kv_cache_manager), - (self.kv_cache_block_offsets, self.draft_kv_cache_block_offsets), - self._steady_block_table_keys): - key = None - if manager is not None: - key = manager.batch_block_table_key(self.request_ids) - if key is None or key != last_key: - manager.copy_batch_block_offsets(block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - num_seqs, - max_blocks=None) - keys.append(key) - self._steady_block_table_keys = tuple(keys) - self._bind_runtime_views( + key = self.kv_cache_manager.batch_block_table_key(self.request_ids) + if key is None or key != self._steady_block_table_key: + self.kv_cache_manager.copy_batch_block_offsets( + self.kv_cache_block_offsets, + self.request_ids, + self.beam_width, + self.num_contexts, + num_seqs, + max_blocks=None) + # A full prepare cleared the key, so a draft's first steady step after + # it copies the full table and records its own manager's key here. + self._steady_block_table_key = key + self.bind_runtime_views( kv_lens_cuda=self.kv_lens_cuda[:num_seqs], kv_lens=host_kv_lens, prompt_lens_cuda=self.prompt_lens_cuda[:num_seqs], prompt_lens_cpu=self.prompt_lens_cpu[:num_seqs], host_request_types=self.host_request_types[:num_seqs], ) + for extension in md._attached_extensions(): + if extension is not self: + extension.prepare() + self.clear_lengths_changed() + + def aligned_kv_lens_staging(self, num_lengths: int, + device: torch.device) -> Optional[torch.Tensor]: + """An int32 buffer of at least ``num_lengths`` entries kept for this metadata. + + Kernels that need aligned length pointers copy a misaligned view (the + generation rows of a mixed batch) into it, so every layer of a step + passes the same pointer. Built on first use, at the request capacity; + None while a CUDA graph is capturing and no such buffer exists yet. + """ + staging = self._aligned_kv_lens_staging + if (staging is None or staging.numel() < num_lengths + or staging.device != torch.device(device)): + if torch.cuda.is_current_stream_capturing(): + return None + capacity = max(int(self.max_num_requests or 0), num_lengths) + staging = torch.empty(capacity, dtype=torch.int32, device=device) + self._aligned_kv_lens_staging = staging + return staging + + def cache_context_plan(self, plan: Optional[_ContextPlan]) -> None: + """Keep ``plan`` as the context-only plan the layers of this forward share. + + The plan is dropped whenever the lengths or the batch can change + (prepare, the runtime views, a draft refresh). + """ + self.context_plan = plan def prepare_encoder_only(self) -> None: """Fast path for encoder-only forward (eager + CUDA graph capture).""" extra_attrs = get_model_extra_attrs() if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(self) + get_global_attrs().attention_metadata = weakref.ref(self.metadata) # Encoder batches run without a KV cache. The block-offset / block-id # attributes default to None (declared as dataclass fields). self.kv_cache_params = KVCacheParams(use_cache=False) + self._invalidate_mla_scheduler_buffers() # For encoder batches every request is a context request, so total # kv-tokens equals total q-tokens. @@ -1436,7 +1636,7 @@ def prepare_encoder_only(self) -> None: # Graph metadata binds these views once per key; eager refreshes them # because batch shape can vary between calls. if not self.is_cuda_graph: - self._bind_runtime_views( + self.bind_runtime_views( kv_lens_cuda=self._seq_lens_cuda[:n], kv_lens=self._seq_lens[:n], prompt_lens_cuda=self._seq_lens_cuda[:n], @@ -1444,15 +1644,31 @@ def prepare_encoder_only(self) -> None: host_request_types=self.host_request_types[:n], ) - def bind_encoder_cuda_graph_seq_lens(self, seq_lens_host: torch.Tensor, - padded_batch_size: int) -> None: - """Bind stable seq_lens storage for one encoder CUDA graph key.""" + def bind_encoder_cuda_graph_seq_lens( + self, + seq_lens_host: torch.Tensor, + padded_batch_size: int, + *, + max_q_len: int, + offsets: Optional[torch.Tensor] = None) -> None: + """Bind stable seq_lens storage for one encoder CUDA graph key. + + ``max_q_len`` fixes the FMHA launch parameters for every replay of the + key. ``offsets`` (device, int32, ``[padded_batch_size + 1]``) are the + fixed query and KV token boundaries of the graph's sequence slots; + None keeps ``cumsum(seq_lens)``. + """ + self.max_context_q_len_override = max_q_len self._seq_lens = seq_lens_host[:padded_batch_size] self._num_contexts = padded_batch_size self._num_generations = 0 + self.request_ids = list(range(padded_batch_size)) + if offsets is not None: + self.cu_q_seqlens = offsets + self.cu_kv_seqlens = offsets self.host_request_types[:padded_batch_size].fill_(0) - self._bind_runtime_views( + self.bind_runtime_views( kv_lens_cuda=self._seq_lens_cuda[:padded_batch_size], kv_lens=self._seq_lens, prompt_lens_cuda=self._seq_lens_cuda[:padded_batch_size], @@ -1460,6 +1676,7 @@ def bind_encoder_cuda_graph_seq_lens(self, seq_lens_host: torch.Tensor, host_request_types=self.host_request_types[:padded_batch_size], ) self.host_total_kv_lens[1] = 0 + self.clear_lengths_changed() def prepare_encoder_cuda_graph_replay(self, seq_lens: List[int], padded_num_tokens: int) -> None: @@ -1469,23 +1686,6 @@ def prepare_encoder_cuda_graph_replay(self, seq_lens: List[int], self._num_ctx_tokens = padded_num_tokens self.host_total_kv_lens[0] = padded_num_tokens - def prepare_flash_mla(self) -> None: - self._flash_mla_metadata_valid = False - # Request-specific fills and H2D copies must happen before replay, not - # become fixed operations in the captured graph. - if torch.cuda.is_current_stream_capturing(): - return - - block_ids_per_seq = maybe_pin_memory( - self.kv_cache_manager.get_block_ids_per_seq(self.request_ids)) - num_blocks = block_ids_per_seq.shape[1] - self.kv_block_ids_per_seq.fill_(0) - self.kv_block_ids_per_seq[:self.num_seqs, :num_blocks].copy_( - block_ids_per_seq, non_blocking=True) - self.block_ids_per_seq.fill_(0) - self.block_ids_per_seq[:self.num_generations, :num_blocks].copy_( - block_ids_per_seq[self.num_contexts:], non_blocking=True) - def pre_process_for_chunked_prefill( self, chunked_seq_len: torch.Tensor, @@ -1707,206 +1907,91 @@ def prepare_context_mla_with_cached_kv(self, self.ctx_kv_indptr[:self.num_contexts + 1].copy_( self.host_ctx_kv_indptr[:self.num_contexts + 1], non_blocking=True) - def spec_decoding_param_prepare_for_blackwell(self) -> None: - """ - Prepare the blackwell parameters for the speculative decoding (Medusa and Eagle) generation-phase attention kernels. - Uses persistent buffers (only allocate if None) for CUDA graph compatibility. - """ - if self.spec_decoding_bl_tree_mask_offset is None: - self.spec_decoding_bl_tree_mask_offset = torch.zeros( - [self.max_num_requests], - dtype=torch.int64, - device='cuda', - ) - if self.spec_bl_tree_first_sparse_mask_offset_kv is None: - self.spec_bl_tree_first_sparse_mask_offset_kv = torch.zeros( - [self.max_num_requests], - dtype=torch.int32, - device='cuda', - ) +class TrtllmAttentionMetadataNativeFields: + """The metadata members the native attention op reads, with their types. - if self.spec_decoding_bl_tree_mask is None: - seqLenQ = self.spec_decoding_packed_mask.shape[ - 1] if self.spec_decoding_packed_mask is not None else 1 - mask_size = torch.ops.trtllm.spec_dec_tree_custom_mask_capacity( - self.max_num_requests, seqLenQ, self.max_num_heads_per_kv) - self.spec_decoding_bl_tree_mask = torch.zeros( - mask_size, - dtype=torch.uint32, - device='cuda', - ) + The native op reads ``meta.`` of the metadata by attribute name. + scripts/generate_fmha_params.py mirrors TrtllmAttentionMetadata into the + native AttentionMetadataParams from these annotations, because the + metadata's members live on its core and extensions. Every name resolves on + every TrtllmAttentionMetadata: the core's and the attached extensions' + fields are forwarded, and TrtllmAttentionMetadata answers the Helix and + FlashMLA members with None while those extensions are not attached. + """ - def update_blackwell_first_sparse_mask_offset(self) -> None: - """Fill gen slots [0:ng) with (kv_lens_cuda - seq_lens); in-place for CUDA graph.""" - nc = self.num_contexts - n = self.num_seqs - ng = n - nc - if ng > 0: - gen_offset = (self.kv_lens_cuda[nc:n] - - self._seq_lens_cuda[nc:n]).to(torch.int32) - self.spec_bl_tree_first_sparse_mask_offset_kv[:ng].copy_(gen_offset) + max_num_requests: int + max_num_sequences: Optional[int] + cache_indirection: Optional[torch.Tensor] + is_cross: bool + effective_beam_width: int + max_seq_len: int + max_context_q_len_override: Optional[int] + is_spec_decoding_enabled: bool + is_spec_dec_tree: bool + force_prepare_spec_dec_tree_mask: bool + max_total_draft_tokens: Optional[int] + trtllm_gen_jit_warmup: bool + helix_position_offsets: Optional[torch.Tensor] + helix_is_inactive_rank: Optional[torch.Tensor] + kv_cache_block_offsets: Optional[torch.Tensor] + block_ids_per_seq: Optional[torch.Tensor] + flash_mla_tile_scheduler_metadata: Optional[torch.Tensor] + flash_mla_num_splits: Optional[torch.Tensor] + use_paged_context_fmha: bool + num_sparse_topk: int + max_context_length: int + host_kv_cache_pool_pointers: Optional[torch.Tensor] + host_kv_cache_pool_mapping: Optional[torch.Tensor] + kv_lens_runtime: torch.Tensor + prompt_lens_cpu_runtime: torch.Tensor + + +# Extension fields the native op reads by name, and their value while the +# declaring extension is not attached. +_NATIVE_FIELDS_OF_ABSENT_EXTENSIONS = { + "helix_position_offsets": None, + "helix_is_inactive_rank": None, + "flash_mla_tile_scheduler_metadata": None, + "flash_mla_num_splits": None, + "is_spec_decoding_enabled": False, + "is_spec_dec_tree": False, + "max_total_draft_tokens": None, +} - def update_spec_dec_param( - self, - batch_size, - is_spec_decoding_enabled, - is_spec_dec_tree, - is_spec_dec_dynamic_tree, - max_draft_len, - max_total_draft_tokens, - spec_metadata: Optional['SpecMetadata'] = None, - spec_tree_manager: Optional['SpecTreeManager'] = None, - num_contexts: int = 0, - ) -> None: - ''' - Update the spec-dec parameters for the TRTLLM attention layer. - Args: - batch_size: int, the number of requests in the batch. - is_spec_decoding_enabled: bool, whether the attention need to be spec_decoding mode, which is determined by attention_need_spec_dec_mode() function. - is_spec_dec_tree: bool, whether the spec-dec mode is a tree, i.e., dynamic tree. For linear-tree, it is always False. - is_spec_dec_dynamic_tree: bool, whether using dynamic tree. - max_draft_len: int, the number of the draft layers. - max_total_draft_tokens: int, the number of all nodes in the tree (except the root). - spec_metadata: Optional['SpecMetadata'] = None, the metadata of the spec-dec. - spec_tree_manager: Optional['SpecTreeManager'] = None, the spec_tree_manager for draft token tree. - num_contexts: int = 0, the number of context (prefill) requests in the - batch. The slot-storage layout is ``[ctx | gen]``; dynamic-tree - metadata only describes the gen rows, so we must source from - ``[num_contexts:batch_size]`` rather than ``[:batch_size]``. - ''' - self.context_plan = None - # Blackwell trtllm-gen spec-dec is enabled only for dynamic-tree masks. - self.is_spec_decoding_enabled = is_spec_decoding_enabled and ( - not self.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()) - or is_spec_dec_dynamic_tree) - - # use_spec_decoding is default to true by default, change in runtime by layers / requests - self.use_spec_decoding = self.is_spec_decoding_enabled - self.is_spec_dec_tree = is_spec_dec_tree - self.is_spec_dec_dynamic_tree = is_spec_dec_dynamic_tree - # A hybrid model's first executed attention layer can have a nonzero - # cache-local index because recurrent layers precede it. Do not rely - # on the C++ ``layer_idx == 0`` fallback to rebuild the target mask: - # the dynamic draft loop clears that mask before the next target step. - self.force_prepare_spec_dec_tree_mask = is_spec_dec_dynamic_tree - # Forward the tree length to FMHA kernel selection. - self.max_total_draft_tokens = max_total_draft_tokens - - # Parameters can be fixed and not changed during runtime if the - if self.is_spec_decoding_enabled: - # Skip pre-allocating position_offsets and packed_mask when dynamic draft length is enabled. - # We will use per-draft-len cached tensors for position_offsets and packed_mask instead. - # Currently dynamic draft length is only supported for linear tree (not is_spec_dec_tree). - - # These buffers are accessed more like removing input padding, - # rather than using max_total_draft_tokens + 1 as the offset between different requests. - if is_spec_dec_tree and self.spec_decoding_position_offsets is None: - # Dynamic tree: use _internal_buf_dim which may be larger - # than max_total_draft_tokens+1 to accommodate K*max_draft_len. - # 1D layout for flexible view() in the drafting loop. - buf_dim = spec_tree_manager._internal_buf_dim - self.spec_decoding_position_offsets = torch.empty( - (self.max_num_requests * buf_dim, ), - dtype=torch.int, - device='cuda', - ) - if is_spec_dec_tree and self.spec_decoding_packed_mask is None: - buf_dim = spec_tree_manager._internal_buf_dim - # Zero-init: dynamic-tree dst has inner dim - # ceil(buf_dim/32) but only ceil((max_total_draft_tokens+1)/32) - # is written each step. Unwritten cols would otherwise feed the - # C++ XQA kernel as live mask bits. - self.spec_decoding_packed_mask = torch.zeros( - [self.max_num_requests, buf_dim, - math.ceil(buf_dim / 32)], - dtype=torch.int, - device='cuda', - ) +class TrtllmAttentionMetadata(AttentionMetadata): + core_metadata_type: ClassVar[ + Type[AttentionCoreMetadata]] = TrtllmCoreMetadata + # The attention op reads cu_q_seqlens / cu_kv_seqlens as given. + supports_explicit_offsets: ClassVar[bool] = True + # The device KV lengths take the step's corrections in prepare. + supports_kv_lens_offsets: ClassVar[bool] = True - if self.spec_decoding_generation_lengths is None: - self.spec_decoding_generation_lengths = torch.empty( - [self.max_num_requests], - dtype=torch.int, - device='cuda', - ) + # Tensors produced and consumed inside the captured graph; the strict + # buffer check skips them. + graph_temporary_attrs: ClassVar[FrozenSet[str]] = frozenset() - if self.is_sm_version_trtllm_gen_kernel(sm=get_sm_version()): - self.spec_decoding_param_prepare_for_blackwell() - else: - self.spec_decoding_bl_tree_mask_offset = None - self.spec_decoding_bl_tree_mask = None - self.spec_bl_tree_first_sparse_mask_offset_kv = None - - # Case 1: dynamic tree — copy per-request params from spec_tree_manager. - if self.is_spec_dec_dynamic_tree: - assert spec_tree_manager is not None, "spec_tree_manager is required for dynamic tree" - n_dt = spec_tree_manager.max_total_draft_tokens + 1 - buf_dim = spec_tree_manager._internal_buf_dim - - assert buf_dim >= n_dt, ( - f"Dynamic-tree copy requires buf_dim >= n_dt; got " - f"buf_dim={buf_dim}, n_dt={n_dt}.") - - slot_storage = spec_tree_manager.slot_storage - - num_gens = batch_size - num_contexts - if num_gens > 0: - slot_ids = slot_storage.all_ids_buf[num_contexts:batch_size] - - pos_src = torch.index_select(slot_storage.position_offsets, - 0, slot_ids)[:, :n_dt] - pos_dst = self.spec_decoding_position_offsets[:num_gens * - n_dt].view( - num_gens, - n_dt) - pos_dst.copy_(pos_src, non_blocking=True) - - actual_mask_width = math.ceil(n_dt / 32) - mask_src = torch.index_select( - slot_storage.packed_mask, 0, - slot_ids)[:, :, :actual_mask_width] - if self.is_sm_version_trtllm_gen_kernel( - sm=get_sm_version()): - # Blackwell reads the padded 3D mask layout. - self.spec_decoding_packed_mask[:num_gens, :n_dt, : - actual_mask_width].copy_( - mask_src, - non_blocking=True) - else: - # Hopper XQA reads a compact flat prefix. - total = num_gens * n_dt * actual_mask_width - self.spec_decoding_packed_mask.view(-1)[:total].copy_( - mask_src.reshape(-1), non_blocking=True) - - self.spec_decoding_generation_lengths[:batch_size].fill_(n_dt) - self.spec_decoding_query_len = n_dt - - # Case 2: linear tree - else: - # Currently dynamic draft length is only supported for linear tree - # Dynamic draft length needs position offsets and packed mask to be shaped for each runtime draft length. - # So we create cache for position offsets and packed mask for each draft length to avoid reallocation. - assert max_draft_len == max_total_draft_tokens, "max_draft_len should be equal to max_total_draft_tokens for linear tree" - # For algos other than PARD, this equals runtime_draft_len (K); for PARD it's 2K-1. - runtime_draft_token_buffer_width = ( - spec_metadata.runtime_tokens_per_gen_step - - 1 if spec_metadata is not None else max_draft_len) - self.generate_spec_decoding_generation_length( - runtime_draft_len=runtime_draft_token_buffer_width) - self.spec_decoding_position_offsets = generate_spec_decoding_position_offsets( - self.max_num_requests, runtime_draft_token_buffer_width) - self.spec_decoding_packed_mask = generate_spec_decoding_packed_mask( - self.max_num_requests, runtime_draft_token_buffer_width) - self.spec_decoding_query_len = runtime_draft_token_buffer_width + 1 - - def generate_spec_decoding_generation_length(self, runtime_draft_len): - self.spec_decoding_generation_lengths[:self.max_num_requests].fill_( - runtime_draft_len + 1) - - def is_sm_version_trtllm_gen_kernel(self, sm): - return is_sm_100f(sm) + @classmethod + def default_extension_types(cls, settings: dict[str, Any]) -> list: + types = [ + *super().default_extension_types(settings), SpecDecodingMetadata + ] + mapping = settings.get("mapping") + if mapping is not None and mapping.has_cp_helix(): + types.append(HelixMetadata) + if callable( + getattr(settings.get("kv_cache_manager"), + "get_fp4_mla_page_table_spec", None)): + types.append(Fp4MlaState) + return types + + def __getattr__(self, name: str) -> Any: + # Reached only when normal lookup fails, so only while the extension + # declaring a native-read field is not attached. + if name in _NATIVE_FIELDS_OF_ABSENT_EXTENSIONS: + return _NATIVE_FIELDS_OF_ABSENT_EXTENSIONS[name] + return super().__getattr__(name) class TrtllmAttention(AttentionBackend[TrtllmAttentionMetadata]): @@ -2327,31 +2412,6 @@ def get_quantize_output_dtype( return torch.float8_e4m3fn return None - def _compute_flash_mla_metadata(self, - metadata: TrtllmAttentionMetadata) -> None: - num_generations = metadata.num_generations - if num_generations <= 0: - return - - generation_slice = slice(metadata.num_contexts, - metadata.num_contexts + num_generations) - generation_kv_lens = metadata.kv_lens_cuda_runtime[generation_slice] - generation_num_splits = metadata.flash_mla_num_splits[:num_generations + - 1] - - s_q = int(metadata.seq_lens[metadata.num_contexts].item()) - - thop.compute_flash_mla_metadata( - generation_kv_lens, - metadata.flash_mla_tile_scheduler_metadata, - generation_num_splits, - num_generations, - s_q, - self.num_heads, - 1, - self.kv_lora_rank, - ) - def _quantize_sage_inputs( self, q: torch.Tensor, @@ -2572,6 +2632,9 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: """Execute the TRTLLM attention backend.""" forward_args = merge_attention_forward_args(forward_args, kwargs) + if isinstance(metadata, AttentionMetadata): + # No forward on lengths changed in place without a refresh. + metadata.check_lengths_current() has_variable_window_starts = (forward_args.variable_window_token_starts is not None) has_variable_window_ends = forward_args.variable_window_token_ends is not None @@ -2591,10 +2654,11 @@ def forward( metadata, TrtllmAttentionMetadata, ) - token_major_view = (metadata.token_major_gen_view() + token_major_view = (metadata.token_major_view() if forward_args.attention_input_type == AttentionInputType.generation_only else None) - if metadata.enable_flash_mla and token_major_view is not None: + if (metadata.find_ext(FlashMlaMetadata) is not None + and token_major_view is not None): raise ValueError( "FlashMLA does not support ragged token-major generation; " "disable FlashMLA for this layout.") @@ -2752,12 +2816,12 @@ def forward( # The flag is invalidated whenever FlashMLA inputs change. The metadata # must always match the compacted generation sub-batch, which is also # the layout used by block_ids_per_seq. - if (metadata.enable_flash_mla and forward_args.attention_input_type + flash_mla = metadata.find_ext(FlashMlaMetadata) + if (flash_mla is not None and forward_args.attention_input_type != AttentionInputType.context_only - and metadata.num_generations > 0 - and not metadata._flash_mla_metadata_valid): - self._compute_flash_mla_metadata(metadata) - metadata._flash_mla_metadata_valid = True + and metadata.num_generations > 0): + flash_mla.compute_scheduler_metadata(self.num_heads, + self.kv_lora_rank) # Blackwell first_sparse: refresh at layer 0 before kernel launch. if self.get_local_layer_idx(metadata) == 0 and ( @@ -3013,7 +3077,7 @@ def _build_context_plan(self, q: torch.Tensor, or metadata.spec_decoding_bl_tree_mask is not None or metadata.spec_bl_tree_first_sparse_mask_offset_kv is not None or metadata.helix_position_offsets is not None - or metadata.enable_flash_mla): + or metadata.find_ext(FlashMlaMetadata) is not None): return plan from .fmha.flashinfer_trtllm_gen import FlashInferTrtllmGenFmha fmha = self._fmha_manager.select(self, q, None, None, metadata, fwd) @@ -3290,8 +3354,8 @@ def append_fp4_mla_context_cache( if not self.uses_fp4_mla_attention or self.mla_params is None: raise RuntimeError( "FP4 MLA context append requires dense FP4 MLA attention.") - if (metadata.fp4_mla_state is None - or metadata.fp4_mla_state.positions is None): + fp4_mla_state = metadata.find_ext(Fp4MlaState) + if fp4_mla_state is None or fp4_mla_state.positions is None: raise RuntimeError( "FP4 MLA context append requires token positions.") self._ensure_rope_table_size(metadata.max_seq_len) @@ -3522,8 +3586,11 @@ def mla_rope_generation( helix_tensor_params = [ metadata.helix_position_offsets, metadata.helix_is_inactive_rank ] + # Declared by the Helix extension, which is attached only under Helix. + helix_spec_tokens_valid = getattr(metadata, "_helix_spec_tokens_valid", + False) if latent_write_target is not None: - if metadata._helix_spec_tokens_valid or kv_norm_weight is not None: + if helix_spec_tokens_valid or kv_norm_weight is not None: raise NotImplementedError( "A latent write target needs the per-token write slots, which Helix " "speculative verify groups and the fused KV norm do not leave free" @@ -3531,7 +3598,7 @@ def mla_rope_generation( # Each request's one-token page is the target row; the per-token slot # 0 addresses it, and -1 skips the token. helix_tensor_params.append(latent_write_target.write_slots) - elif metadata._helix_spec_tokens_valid: + elif helix_spec_tokens_valid: # Speculative verify groups: per-token KV write slots (-1 = this # rank does not own the token's position). The append kernel then # gates and addresses per token instead of per sequence. @@ -3568,7 +3635,7 @@ def mla_rope_generation( @contextlib.contextmanager def presented_token_major_for( self, metadata: TrtllmAttentionMetadata) -> Iterator[None]: - with metadata.presented_token_major(metadata.token_major_gen_view()): + with metadata.presented_token_major(metadata.token_major_view()): yield def _mla_rope_generation_impl( diff --git a/tensorrt_llm/_torch/attention/backends/vanilla.py b/tensorrt_llm/_torch/attention/backends/vanilla.py index aeafe103d672..29baef371ccb 100644 --- a/tensorrt_llm/_torch/attention/backends/vanilla.py +++ b/tensorrt_llm/_torch/attention/backends/vanilla.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import math -from typing import Optional +from typing import ClassVar, Optional, Type import torch import torch.nn.functional as F @@ -15,6 +15,7 @@ except ImportError: AttentionMaskConverter = None +from .core import AttentionCoreMetadata from .interface import (AttentionBackend, AttentionForwardArgs, AttentionInputType, AttentionMask, AttentionMetadata, PredefinedAttentionMask, merge_attention_forward_args) @@ -61,18 +62,34 @@ def generate_sliding_window_mask(batch_size: int, target_length: int, return attention_mask -class VanillaAttentionMetadata(AttentionMetadata): +class VanillaCoreMetadata(AttentionCoreMetadata): + """Vanilla attention state: each sequence's cache block IDs.""" + + # Block IDs, plus the no-KV-cache sequence bound callers set for + # encoder-only use and the page size a sparse algorithm (RocketKV) sets. + fields = frozenset({"block_ids_per_seq", "max_seq_len", "page_size"}) + # The sequence bound of no-KV-cache attention. + init_fields = { + "max_seq_len": None, + } - def __post_init__(self) -> None: - super().__post_init__() + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # Per-step block IDs that derive() creates. + self.block_ids_per_seq = None self.kv_layout = "NHD" - def prepare(self) -> None: - super().prepare() + def derive(self) -> None: + md = self.metadata # indices of used cache blocks for each sequence - assert self.request_ids is not None - self.block_ids_per_seq = self.kv_cache_manager.get_batch_cache_indices( - self.request_ids) if self.kv_cache_manager is not None else None + assert md.request_ids is not None + self.block_ids_per_seq = md.kv_cache_manager.get_batch_cache_indices( + md.request_ids) if md.kv_cache_manager is not None else None + + +class VanillaAttentionMetadata(AttentionMetadata): + core_metadata_type: ClassVar[ + Type[AttentionCoreMetadata]] = VanillaCoreMetadata class VanillaAttention(AttentionBackend[VanillaAttentionMetadata]): @@ -794,6 +811,9 @@ def forward(self, forward_args: Optional[AttentionForwardArgs] = None, **kwargs) -> torch.Tensor: forward_args = merge_attention_forward_args(forward_args, kwargs) + if isinstance(metadata, AttentionMetadata): + # No forward on lengths changed in place without a refresh. + metadata.check_lengths_current() if metadata.multi_item_part_lens is not None: raise ValueError( diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv4.py b/tensorrt_llm/_torch/models/modeling_deepseekv4.py index bb8308465e6f..52d674dbbd2a 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv4.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv4.py @@ -2869,6 +2869,8 @@ def forward( remote_tail_at = self._remote_tail_boundary(attn_metadata) engram_kv_cache: dict[int, torch.Tensor] = {} + # The metadata the layers run on; a replayed suffix runs on its own. + layer_metadata = attn_metadata for idx, decoder_layer in enumerate(self.layers[: self.num_hidden_layers]): if idx == remote_tail_at: boundary_output, hc_state = self._enter_remote_tail_boundary( @@ -2877,7 +2879,7 @@ def forward( if boundary_output is not None: return boundary_output if idx == replay_at: - position_ids, input_ids, hc_state = self._enter_bounded_replay( + position_ids, input_ids, hc_state, layer_metadata = self._enter_bounded_replay( replay_plan, attn_metadata, position_ids, input_ids, hc_state ) if image_mask is not None and replay_plan.replays_local_tokens: @@ -2905,7 +2907,7 @@ def forward( hc_state = decoder_layer( position_ids=position_ids, hc_state=hc_state, - attn_metadata=attn_metadata, + attn_metadata=layer_metadata, spec_metadata=spec_metadata, input_ids=input_ids, engram_embeddings=engram_embeddings, @@ -2991,7 +2993,8 @@ def _enter_bounded_replay(self, plan, attn_metadata, position_ids, input_ids, hc """Re-index the per-token state the replayed layers will read. Never called on V4, because :meth:`_plan_bounded_replay` never returns a - split. Returns the three per-token carriers narrowed to the replayed rows. + split. Returns the three per-token carriers narrowed to the replayed rows + and the attention metadata the replayed layers run on. """ raise NotImplementedError @@ -2999,7 +3002,7 @@ def _exit_bounded_replay(self, plan, attn_metadata, hidden_states): """Restore encoder-pass shape and metadata after a successful replayed suffix. Never called on V4. Returns ``hidden_states`` back at the encoder pass's - row count and restores the matching metadata for downstream consumers. + row count; ``attn_metadata`` is the encoder pass's metadata. """ raise NotImplementedError diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv41.py b/tensorrt_llm/_torch/models/modeling_deepseekv41.py index 07628c7456b1..ed177b423901 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv41.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv41.py @@ -38,7 +38,7 @@ plan_decoder_replay, scatter_replayed_rows, ) -from ..attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from ..attention.backends.sparse.csa2.metadata import CSA2Metadata, CSA2TrtllmMetadata from ..attention.backends.sparse.csa2.module import DeepseekV41Attention as CSA2Attention from ..attention.backends.sparse.csa2.params import CSA2Mode from ..configs.deepseek_v41 import ( @@ -778,7 +778,9 @@ def _precompute_engram_kv( def forward(self, attn_metadata, *args, **kwargs): from ..pyexecutor.ced_replay import uses_private_decoder_cache - attn_metadata.begin_model_forward() + csa2 = attn_metadata.find_ext(CSA2Metadata) + if csa2 is not None: + csa2.reset_forward_state() spec_metadata = kwargs.get("spec_metadata") if isinstance(spec_metadata, DSparkSpecMetadata): spec_metadata.context_capture_lens = None @@ -1137,24 +1139,22 @@ def _enter_bounded_replay( position_ids: torch.Tensor | None, input_ids: torch.Tensor | None, hc_state: HCState, - ) -> tuple[torch.Tensor | None, torch.Tensor | None, HCState]: - """Resolve deferred mHC state, then compact token rows and rebuild metadata.""" + ) -> tuple[torch.Tensor | None, torch.Tensor | None, HCState, CSA2TrtllmMetadata]: + """Resolve deferred mHC state, then compact token rows onto the replay metadata.""" from ..pyexecutor.ced_replay import uses_private_decoder_cache if not plan.updates_local_metadata: - enter_decoder_replay(attn_metadata, plan) - return position_ids, input_ids, hc_state + return position_ids, input_ids, hc_state, enter_decoder_replay(attn_metadata, plan) hc_state = self._resolve_hc_state(hc_state) if self.ced_kv_precompute: prepare_ced_global_kv(self.layers[self.decoder_replay_split], hc_state, attn_metadata) - enter_decoder_replay( + replay_metadata = enter_decoder_replay( attn_metadata, plan, self.decoder_replay_split if uses_private_decoder_cache(self, attn_metadata.kv_cache_manager) else None, ) - attn_metadata.csa2_replay_query_rows = plan.rows return ( gather_replayed_rows(position_ids, plan), gather_replayed_rows(input_ids, plan), @@ -1162,10 +1162,11 @@ def _enter_bounded_replay( gather_replayed_rows(hc_state.residual, plan), pre_mix=gather_replayed_rows(hc_state.pre_mix, plan), ), + replay_metadata, ) def _exit_bounded_replay(self, plan, attn_metadata, hidden_states): - """Restore the output layout and metadata after successful Decoder execution.""" + """Restore the output layout after successful Decoder execution.""" hidden_states = scatter_replayed_rows(hidden_states, plan) exit_decoder_replay(attn_metadata, plan) attn_metadata.csa2_decoder_capture_lens = tuple( diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 60a1aea3e113..569ddc2b38f8 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -627,7 +627,9 @@ def build(self, key: EncoderGraphKey) -> AttentionMetadata: metadata.seq_lens = torch.zeros( key.num_contexts, dtype=torch.int32, pin_memory=prefer_pinned() ) - metadata.bind_encoder_cuda_graph_seq_lens(metadata.seq_lens, key.num_contexts) + metadata.bind_encoder_cuda_graph_seq_lens( + metadata.seq_lens, key.num_contexts, max_q_len=None + ) return metadata def refresh_in_place( diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3.py b/tensorrt_llm/_torch/models/modeling_minimaxm3.py index 00267c14f162..a9414da0b49c 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3.py @@ -56,6 +56,7 @@ _write_main_kv_slots_to_pool, ) from ..attention.backends.sparse.minimax_m3.common import get_sparse_layer_ids, index_head_range +from ..attention.backends.sparse.minimax_m3.triton_metadata import MiniMaxM3TritonMetadata from ..attention.backends.sparse.params import SparseBackendForwardArgs from ..attention.backends.trtllm import TrtllmAttention from ..distributed import AllReduce, AllReduceFusionOp, AllReduceParams, MiniMaxAllReduceRMS @@ -409,7 +410,7 @@ def _validate_sparse_attention_runtime_config( if sparse_config is None or sparse_config.algorithm != "minimax_m3": raise ValueError( "MiniMax-M3 requires sparse_attention_config.algorithm='minimax_m3' " - "to create its KV-cache manager and prepare attn_metadata.minimax_m3. " + "to create its KV-cache manager and prepare its attention metadata. " "Set the following in the LLM API configuration:\n" "sparse_attention_config:\n algorithm: minimax_m3" ) @@ -1547,7 +1548,7 @@ def _fused_fp8_index_qk_norm_rope( return torch.ops.trtllm.minimax_m3_fp8_indexer_qk_norm_rope( idx_qk.contiguous(), index_k_cache, - attn_metadata.msa_out_cache_loc[:num_tokens], + attn_metadata.msa_write_slots(num_tokens), self.sparse_num_index_heads, self.sparse_index_dim, rotary_dim, @@ -1596,7 +1597,7 @@ def _fused_fp8_qkv_indexer_norm_rope_kv_insert( if kv_cache_manager.is_nvfp4_layer(self.layer_idx) else None ) - out_cache_loc = getattr(attn_metadata, "msa_out_cache_loc", None) + out_cache_loc = attn_metadata.msa_write_slots() else: buffers, index_k_cache, scale_buffers, out_cache_loc = cache_tensors if out_cache_loc is None: @@ -1973,17 +1974,17 @@ def _sdpa_dense_attention_core( # 5. Read the pre-built M3 metadata. Building is the # responsibility of ``MiniMaxM3AttentionMetadata.prepare`` (or # the test path), so the forward stays CUDA-graph-capture safe. - m3_attachment = getattr(attn_metadata, "minimax_m3", None) - if m3_attachment is None: + m3 = attn_metadata.find_ext(MiniMaxM3TritonMetadata) + if m3 is None or m3.sparse_metadata is None: raise RuntimeError( f"MiniMax-M3 dense forward (layer {self.layer_idx}) requires " - "attn_metadata.minimax_m3 to be pre-built by " - "MiniMaxM3AttentionMetadata.prepare(); the model_engine " - "configures the M3 backend's Metadata class so this happens " - "automatically. Test callers must attach minimax_m3 manually." + "the MiniMaxM3TritonMetadata extension to be prepared; the " + "model_engine attaches it through the M3 backend's " + "metadata_extensions so this happens automatically. Test " + "callers must attach and fill it manually." ) - m3_meta = m3_attachment["metadata"] - out_cache_loc = m3_attachment["out_cache_loc"] + m3_meta = m3.sparse_metadata + out_cache_loc = m3.out_cache_loc # 6. Write the new tokens' K/V to the pool. All inputs come # from prepare() which lands them on the cache device, so the @@ -2288,11 +2289,11 @@ def _sparse_forward( 8. Apply ``o_proj`` and return. Production callers (the LLM API path) drive - :meth:`MiniMaxM3AttentionMetadata.prepare` outside any - CUDA-graph capture window; that method attaches a pre-built + :meth:`MiniMaxM3TritonMetadata.prepare` outside any + CUDA-graph capture window; that method stores a pre-built :class:`MiniMaxM3TritonSparseAttentionMetadata` and an - ``out_cache_loc`` tensor as ``attn_metadata.minimax_m3``. Test - callers attach the same dict manually. This forward path + ``out_cache_loc`` tensor on the extension. Test callers fill + the same extension manually. This forward path always reads the pre-built attachment and never builds metadata itself — both would trigger ``cudaErrorStreamCaptureUnsupported`` for the @@ -2330,7 +2331,7 @@ def _sparse_forward( kv_cache, index_k_cache, kv_scale_cache, - attn_metadata.msa_out_cache_loc, + attn_metadata.msa_write_slots(), self.layer_idx_str, ) o = self._forward_attention_core(q, None, None, idx_q, None, attn_metadata) @@ -2515,17 +2516,17 @@ def _triton_sparse_attention_core( # 5. Read the pre-built M3 metadata. Building is the # responsibility of ``MiniMaxM3AttentionMetadata.prepare`` (or # the test path), so the forward stays CUDA-graph-capture safe. - m3_attachment = getattr(attn_metadata, "minimax_m3", None) - if m3_attachment is None: + m3 = attn_metadata.find_ext(MiniMaxM3TritonMetadata) + if m3 is None or m3.sparse_metadata is None: raise RuntimeError( f"MiniMax-M3 sparse forward (layer {self.layer_idx}) requires " - "attn_metadata.minimax_m3 to be pre-built by " - "MiniMaxM3AttentionMetadata.prepare(); the model_engine " - "configures the M3 backend's Metadata class so this happens " - "automatically. Test callers must attach minimax_m3 manually." + "the MiniMaxM3TritonMetadata extension to be prepared; the " + "model_engine attaches it through the M3 backend's " + "metadata_extensions so this happens automatically. Test " + "callers must attach and fill it manually." ) - m3_meta = m3_attachment["metadata"] - out_cache_loc = m3_attachment["out_cache_loc"] + m3_meta = m3.sparse_metadata + out_cache_loc = m3.out_cache_loc if not self.disable_index_value and idx_v_cache is not None: # The shared idx_v_proj is not part of the M3 checkpoint diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_next.py b/tensorrt_llm/_torch/models/modeling_qwen3_next.py index c66d3b860425..e89763aa06ee 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_next.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_next.py @@ -1169,13 +1169,8 @@ def forward( attn_metadata: AttentionMetadata, all_rank_num_tokens: Optional[List[int]] = None, spec_metadata: Optional[SpecMetadata] = None, - kv_lens_increment: int = 0, **kwargs, ) -> torch.Tensor: - """``kv_lens_increment``: ``attn_metadata.kv_lens_cuda[:n] += 1`` with - ``n = kv_lens_increment``, deferred by the draft loop to this pass and - applied before anything reads kv_lens (inside the fused input launch - when this pass takes it).""" # Install this draft step's attention-DP token distribution: the draft # loop passes it as a kwarg and leaves attn_metadata alone, matching # Eagle3DraftModel.forward. The MoE below reads the per-rank counts off @@ -1198,21 +1193,13 @@ def forward( # the row-parallel chunk in one launch (modules/q35_mtp_glue.py). from ..modules.q35_mtp_glue import mtp_fused_input hidden_states = mtp_fused_input( - input_ids, - embed_tokens.weight, - hidden_states, + input_ids, embed_tokens.weight, hidden_states, self.pre_fc_norm_embedding.weight, self.pre_fc_norm_hidden.weight, self.pre_fc_norm_embedding.variance_epsilon, self.pre_fc_norm_hidden.variance_epsilon, - tp_size if chunked else 1, - tp_rank if chunked else 0, - kv_lens=attn_metadata.kv_lens_cuda - if kv_lens_increment else None, - kv_rows=kv_lens_increment) + tp_size if chunked else 1, tp_rank if chunked else 0) else: - if kv_lens_increment: - attn_metadata.kv_lens_cuda[:kv_lens_increment] += 1 hidden_states = self._fc_input(input_ids, hidden_states, embed_tokens, chunked, tp_size, tp_rank) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index 96107532b165..021dac6afde5 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -1348,7 +1348,7 @@ def build(self, key: EncoderGraphKey) -> AttentionMetadata: metadata.seq_lens = torch.ones(key.num_contexts, dtype=torch.int32) bind_seq_lens = getattr(metadata, "bind_encoder_cuda_graph_seq_lens", None) if callable(bind_seq_lens): - bind_seq_lens(metadata.seq_lens, key.num_contexts) + bind_seq_lens(metadata.seq_lens, key.num_contexts, max_q_len=None) return metadata def refresh_in_place( diff --git a/tensorrt_llm/_torch/models/modeling_qwen4_exp.py b/tensorrt_llm/_torch/models/modeling_qwen4_exp.py index c2e54cba74ec..162121aba891 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen4_exp.py +++ b/tensorrt_llm/_torch/models/modeling_qwen4_exp.py @@ -31,6 +31,7 @@ register_input_processor, ) from ..attention.backends import AttentionMetadata +from ..attention.backends.extensions import MetadataExtension from ..attention.backends.sparse.qsa.indexer import QSAIndexer from ..attention.backends.sparse.qsa.params import QSASparseParams from ..distributed import AllReduce, AllReduceParams, allgather @@ -76,6 +77,25 @@ def _qwen4_exp_tp_output_reduction_enabled(mapping) -> bool: return mapping.tp_size > 1 and not mapping.enable_attention_dp +class Qwen4ExpPleState(MetadataExtension): + """PLE pools given to the model's forwards without a cache manager. + + ``qwen4_exp_ple_state`` maps a PLE layer index to its (short-conv state, + n-gram context) pools; a test harness that owns its state tensors gives + them at creation (``extension_types=(Qwen4ExpPleState,)``, + ``qwen4_exp_ple_state=...``). Without them the model resolves pools of its + own. + """ + + fields = frozenset( + { + "qwen4_exp_ple_state", + } + ) + methods = frozenset() + init_fields = {"qwen4_exp_ple_state": None} + + def _validate_qwen4_exp_runtime_config( config: PretrainedConfig, layer_types: List[str], @@ -497,6 +517,9 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): # Shapes match the model-owned ``extract_qwen4_exp_ple_cache_params``. # (conv_state_shape / ngram_context_len). self._ple_conv_state: Optional[torch.Tensor] = None + # The pools each PLE layer's last forward resolved, by layer index: + # later forwards and graph capture read the same tensors. + self._resolved_ple_pools: Dict[int, tuple] = {} self._ple_ngram_context: Optional[torch.Tensor] = None # The final Hyper-Connection mixer IS the last norm (use_combine=False); @@ -702,11 +725,10 @@ def _prepare_ple_state( conv_state.index_fill_(0, ctx_slots, 0) ngram_context.index_fill_(0, ctx_slots, self.eos_token_id) - # Surface the resolved pools on metadata so graph capture and the - # decoder layer observe the same tensors. - # The cache-manager branch below is checked first each forward, so this - # write never shadows the authoritative KVCacheManagerV2-owned pools. - attn_metadata.qwen4_exp_ple_state = {ple_layer_idx: (conv_state, ngram_context)} + # Later forwards and graph capture resolve the same pools. The + # cache-manager branch is checked first each forward, so these never + # shadow the authoritative KVCacheManagerV2-owned pools. + self._resolved_ple_pools[ple_layer_idx] = (conv_state, ngram_context) return ple_meta, conv_state, ngram_context def _resolve_ple_pools( @@ -744,6 +766,9 @@ def _resolve_ple_pools( conv_state, ngram_context = provided[ple_layer_idx] self._assert_ple_pool_shape(conv_state, ngram_context, ple_module) return conv_state, ngram_context + resolved = self._resolved_ple_pools.get(ple_layer_idx) + if resolved is not None: + return resolved max_slots = int(attn_metadata.max_num_requests or 0) needed = max(max_slots, num_seq, 1) diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index 4bcf69662c89..cbc95701233c 100644 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -1388,9 +1388,6 @@ def __init__( model_type = model_config.pretrained_config.model_type mtp_layer = None lm_head_quant_config = None - # Whether the MTP layer's forward takes the draft loop's deferred - # ``kv_lens_increment``. - folds_kv_lens_increment = False # A target model that owns its MTP layer class (a Modeling V2 # target, for example) names it, and the drafter builds that class # instead of the built-in one registered for the model type. @@ -1413,7 +1410,6 @@ def __init__( case "qwen3_next" | "qwen3_5_text" | "qwen3_5_moe_text": from .modeling_qwen3_next import Qwen3NextMTP mtp_layer = Qwen3NextMTP - folds_kv_lens_increment = True case "qwen4_exp_text": from .modeling_qwen4_exp import Qwen4ExpMTP mtp_layer = Qwen4ExpMTP @@ -1462,7 +1458,6 @@ def __init__( model.aux_stream_dict) for layer_idx in range(mtp_num_layers) ]) - self.folds_kv_lens_increment = folds_kv_lens_increment dwdp_manager = get_global_dwdp_manager() if dwdp_manager is not None and mtp_repeat_count > 1: for layer_idx in range(mtp_num_layers): diff --git a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py index 1a8c1c9a60ed..08cefb04a269 100644 --- a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py @@ -153,25 +153,10 @@ class KeyType(NamedTuple): ragged_verify_bucket: Optional[int] = None -def _save_spec_decode_capture_state( - attn_metadata: Any, enable_spec_decode: bool) -> Optional[torch.Tensor]: - if not enable_spec_decode or not hasattr(attn_metadata, 'kv_lens_cuda'): - return None - return attn_metadata.kv_lens_cuda[:attn_metadata.num_seqs].clone() - - -def _restore_spec_decode_capture_state( - attn_metadata: Any, saved_kv_lens_cuda: Optional[torch.Tensor]) -> None: - if saved_kv_lens_cuda is None: - return - # Speculative decoding updates kv_lens_cuda in-place during every forward. - # CUDA graph warmup reuses one dummy request for multiple eager forwards, so - # letting those updates accumulate would make later warmups/capture advertise - # more KV tokens than the dummy request actually allocated. Restore the - # single-step input state outside the graph after each forward instead. - batch_size = saved_kv_lens_cuda.shape[0] - attn_metadata.kv_lens_cuda[:batch_size].copy_(saved_kv_lens_cuda) - attn_metadata.on_update_kv_lens() +def _prepare_again(attn_metadata: Any) -> None: + """Prepare ``attn_metadata`` again from the batch it was last prepared with.""" + attn_metadata.prepare(context_requests=attn_metadata.context_requests, + prefill_cuda_graphs=attn_metadata.prefill_cuda_graphs) @dataclass @@ -823,9 +808,15 @@ def capture(self, key: KeyType, forward_fn: Callable, initial_inputs: Dict[str, Any], - enable_spec_decode: bool = False, - postprocess_fn: Optional[Callable] = None) -> Any: - """Warm up and/or capture the forward pass for a graph key.""" + postprocess_fn: Optional[Callable] = None, + prepare_each_forward: bool = False) -> Any: + """Warm up and/or capture the forward pass for a graph key. + + The caller prepared ``initial_inputs["attn_metadata"]`` for the first + warmup forward. With ``prepare_each_forward`` every later warmup + forward and the capture are preceded by a ``prepare`` of the same + batch, so none of them starts from the state a previous forward left. + """ batch_size = key.batch_size # [CUDA graph spec decode padding] # We pad input IDs/position IDs to the maximum draft length (token per request). @@ -874,8 +865,6 @@ def capture(self, capture_inputs[ "encoder_hidden_states"] = static_encoder_hidden_states attn_metadata = capture_inputs["attn_metadata"] - saved_kv_lens_cuda = _save_spec_decode_capture_state( - attn_metadata, enable_spec_decode) self.graph_metadata[key] = { "attn_metadata": attn_metadata, @@ -889,20 +878,20 @@ def capture(self, # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics # This also lets us initialize states in the attn_metadata and # resize the shared attention workspace before any graph is captured. - def run_warmup(): + def run_warmup(step): + if step > 0 and prepare_each_forward: + _prepare_again(attn_metadata) result = forward_fn(capture_inputs) if postprocess_fn is not None: postprocess_fn(capture_inputs) - _restore_spec_decode_capture_state(attn_metadata, - saved_kv_lens_cuda) return result - for _ in range(self.WARMUP_STEPS): + for step in range(self.WARMUP_STEPS): if self.is_warmup_only: - output = run_warmup() + output = run_warmup(step) else: with discard_nccl_window_tensor_outputs(capture_inputs): - run_warmup() + run_warmup(step) if self.is_warmup_only: return output @@ -912,6 +901,8 @@ def run_warmup(): # setup/capture; release its reference before entering. output = None graph_pool = self.memory_pool or torch.cuda.graph_pool_handle() + if self.WARMUP_STEPS > 0 and prepare_each_forward: + _prepare_again(attn_metadata) with nccl_window_graph_capture(graph, graph_pool): # A pre-replay hook reads the static inputs before the replay, so their copies must land before it. prologue_pack = (None if self._pre_replay_hooks else @@ -921,8 +912,6 @@ def run_warmup(): output = forward_fn(capture_inputs) # The addresses the graph baked in, for the strict buffer check. self._record_strict_buffer_snapshot(key) - _restore_spec_decode_capture_state(attn_metadata, - saved_kv_lens_cuda) self.graphs[key] = graph graph_output = make_weak_ref(output) @@ -1100,38 +1089,46 @@ def _snapshot_graph_tensor_ptrs(obj: Any) -> Dict[str, int]: in; any of them changing before a later replay means the graph would read stale (or freed) memory. obj.graph_temporary_attrs are skipped. """ - graph_temporary_attrs = getattr(obj, "graph_temporary_attrs", ()) - return { - name: value.data_ptr() - for name, value in vars(obj).items() - if isinstance(value, torch.Tensor) and value.is_cuda - and name not in graph_temporary_attrs - } + ptrs = {} + for prefix, owner in CUDAGraphRunner._graph_tensor_owners(obj): + graph_temporary_attrs = getattr(owner, "graph_temporary_attrs", ()) + ptrs.update({ + prefix + name: value.data_ptr() + for name, value in vars(owner).items() + if isinstance(value, torch.Tensor) and value.is_cuda + and name not in graph_temporary_attrs + }) + return ptrs + + @staticmethod + def _graph_tensor_owners(obj: Any) -> List[Tuple[str, Any]]: + """obj and, for attention metadata, its extensions, with key prefixes.""" + owners = [("", obj)] + attached = obj.__dict__.get("_extensions") if hasattr( + obj, "__dict__") else None + for extension in attached or (): + owners.append((f"{type(extension).__name__}.", extension)) + return owners @staticmethod def _assert_graph_tensor_ptrs_stable(obj: Any, expected_ptrs: Dict[str, int]) -> None: - """Raise if any tensor captured by _snapshot_graph_tensor_ptrs has moved. - - Attributes in obj.draft_replay_swapped_attrs are deliberately rebound - to draft KV cache buffers for the duration of a draft replay, so their - live value is not the captured one. The target value saved before the - swap is checked in its place. - """ - swapped_attrs = getattr(obj, "draft_replay_swapped_attrs", {}) - for name, expected_ptr in expected_ptrs.items(): - value = (swapped_attrs[name] if name in swapped_attrs else getattr( - obj, name, None)) + """Raise if any tensor captured by _snapshot_graph_tensor_ptrs has moved.""" + owners = dict(CUDAGraphRunner._graph_tensor_owners(obj)) + for key, expected_ptr in expected_ptrs.items(): + prefix, _, name = key.rpartition(".") + owner = owners.get(f"{prefix}." if prefix else "") + value = getattr(owner, name, None) if not isinstance(value, torch.Tensor): raise RuntimeError( f"CUDA graph metadata attribute `{type(obj).__name__}." - f"{name}` was a tensor at capture time but is now " + f"{key}` was a tensor at capture time but is now " f"{type(value).__name__}. Graph-visible buffers must be " "updated in place, not rebound.") if value.data_ptr() != expected_ptr: raise RuntimeError( f"CUDA graph metadata tensor `{type(obj).__name__}." - f"{name}` was reallocated between capture and replay " + f"{key}` was reallocated between capture and replay " f"(data_ptr 0x{expected_ptr:x} -> 0x{value.data_ptr():x}). " "Graph-visible buffers must be updated in place; " "rebinding invalidates addresses baked into the captured " @@ -2567,6 +2564,13 @@ def maybe_get_cuda_graph( key="encoder_cuda_graph_cross_attention_warning") return None, None + if (self.use_fixed_sequence_slots + and not attn_metadata.supports_explicit_offsets): + raise ValueError( + "Encoder CUDA graphs with fixed sequence slots need an attention " + "backend that honors explicit sequence offsets (TRTLLM); " + f"{type(attn_metadata).__name__} derives them from seq_lens") + # First sighting of this key: create graph-resident metadata and bind # it to stable pinned seq_lens storage for future replays. padded_batch_size = len(inputs['seq_lens']) @@ -2580,26 +2584,24 @@ def maybe_get_cuda_graph( ) assert graph_attn_metadata.is_cuda_graph - # Lock FMHA kernel launch params to the padded max_seq_len so the - # cubin + grid dims stay constant across replays for this key. - graph_attn_metadata.max_context_q_len_override = padded_max_seq_len - # Bind graph metadata to stable host seq_lens storage. The storage may # be pinned or pageable; only captured H2D copies require pinned memory. - graph_attn_metadata.bind_encoder_cuda_graph_seq_lens( - self.shared_static_tensors_cpu["seq_lens"], padded_batch_size) + # The FMHA kernel launch params are locked to the padded max_seq_len + # so the cubin + grid dims stay constant across replays for this key. + capture_offsets_cuda = None if self.use_fixed_sequence_slots: # CUDA graph replay keeps each request in its capture-time token # slot. Explicit boundaries let attention combine those fixed - # offsets with the per-replay logical sequence lengths above. - capture_offsets = self._get_capture_sequence_offsets(key) - capture_offsets_cuda = torch.tensor(capture_offsets, - dtype=torch.int32, - device="cuda") - graph_attn_metadata.cu_q_seqlens = capture_offsets_cuda - graph_attn_metadata.cu_kv_seqlens = capture_offsets_cuda - graph_attn_metadata.max_seq_len = self.config.max_seq_len - graph_attn_metadata.request_ids = list(range(padded_batch_size)) + # offsets with the per-replay logical sequence lengths. + capture_offsets_cuda = torch.tensor( + self._get_capture_sequence_offsets(key), + dtype=torch.int32, + device="cuda") + graph_attn_metadata.bind_encoder_cuda_graph_seq_lens( + self.shared_static_tensors_cpu["seq_lens"], + padded_batch_size, + max_q_len=padded_max_seq_len, + offsets=capture_offsets_cuda) self.retire_staging() return graph_attn_metadata, key diff --git a/tensorrt_llm/_torch/pyexecutor/engine/metadata.py b/tensorrt_llm/_torch/pyexecutor/engine/metadata.py index a07748cc0bd8..190e6efdb3f6 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/metadata.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/metadata.py @@ -9,16 +9,20 @@ import torch +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata, RecurrentConfig +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.attention.backends.interface import ( AttentionBackend, AttentionMetadata, AttentionRuntimeFeatures, ) from tensorrt_llm._torch.attention.backends.sparse.hooks import has_sparse_mla_hooks +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.model_config import ModelConfig, _get_kv_cache_layer_specs from tensorrt_llm.mapping import Mapping from ..config_utils import is_mla +from ..kv_cache.mamba_cache_manager import BaseMambaCacheManager if TYPE_CHECKING: from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 @@ -61,8 +65,21 @@ def build_attention_metadata( enable_context_mla_with_cached_kv: bool | None = None, max_num_heads_per_kv: int | None = None, enable_ragged_verification: bool = False, + recurrent_config: RecurrentConfig | None = None, + spec_decoding: SpecDecodingConfig | None = None, + one_model_draft: bool = False, + max_seq_len: int | None = None, + model_extension_types: tuple = (), + model_extension_config: dict | None = None, ) -> AttentionMetadata: - """Construct attention metadata and resolve model-derived inputs.""" + """Construct attention metadata and resolve model-derived inputs. + + ``max_seq_len`` bounds the sequences of no-KV-cache attention; it is given + to the families whose core takes it. ``model_extension_types`` and + ``model_extension_config`` are the extensions the model declares and their + creation arguments (``RuntimeHooks.metadata_extensions`` / + ``metadata_config``), attached with the engine's own. + """ pretrained_config = model_config.pretrained_config sparse_attention_config = model_config.sparse_attention_config sparse_mla = has_sparse_mla_hooks(getattr(sparse_attention_config, "algorithm", None)) @@ -91,27 +108,46 @@ def build_attention_metadata( if sparse_attention_config is not None else None ) - metadata_kwargs = {} + # Ragged verification is taken by the DSA extension, which the TRTLLM + # family attaches for a DSA or DeepSeek-V4 sparse attention config. + if enable_ragged_verification and not issubclass( + attention_backend.Metadata, TrtllmAttentionMetadata + ): + raise ValueError("Ragged verification requires the DSA target metadata backend") + # FlashMLA kernels run on the TRTLLM backend only. + extension_types = ( + (FlashMlaMetadata,) + if model_config.enable_flash_mla + and issubclass(attention_backend.Metadata, TrtllmAttentionMetadata) + else () + ) + # Each argument goes to the extension that takes it: speculative-decoding + # state is TRTLLM's, and recurrent state exists only for hybrid models. + family_settings = {} + if issubclass(attention_backend.Metadata, TrtllmAttentionMetadata): + family_settings["max_num_heads_per_kv"] = max_num_heads_per_kv + if spec_decoding is not None: + family_settings["spec_decoding"] = spec_decoding + if isinstance(kv_cache_manager, BaseMambaCacheManager): + family_settings["recurrent_config"] = recurrent_config + core_type = attention_backend.Metadata.core_metadata_type + if max_seq_len is not None and "max_seq_len" in core_type.declared_init_fields(): + family_settings["max_seq_len"] = max_seq_len if enable_ragged_verification: - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, - ) - - if not issubclass(attention_backend.Metadata, DSAtrtllmAttentionMetadata): - raise ValueError("Ragged verification requires the DSA target metadata backend") - metadata_kwargs["enable_ragged_verification"] = True + family_settings["enable_ragged_verification"] = True return attention_backend.Metadata( max_num_requests=max_batch_size, max_num_tokens=max_num_tokens, max_num_sequences=max_batch_size * max_beam_width, kv_cache_manager=kv_cache_manager, draft_kv_cache_manager=draft_kv_cache_manager, + one_model_draft=one_model_draft, mapping=mapping, runtime_features=attention_runtime_features, - enable_flash_mla=model_config.enable_flash_mla, enable_context_mla_with_cached_kv=enable_context_mla_with_cached_kv, cache_indirection=cache_indirection, - max_num_heads_per_kv=max_num_heads_per_kv, sparse_metadata_params=sparse_metadata_params, - **metadata_kwargs, + extension_types=(*extension_types, *model_extension_types), + **family_settings, + **(model_extension_config or {}), ) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/graphs.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/graphs.py index 1e4d7da8c654..0dc2ed03122c 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/graphs.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/graphs.py @@ -40,11 +40,7 @@ from tensorrt_llm._torch.pyexecutor.sampler.sampler_common import SampleType from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._torch.pyexecutor.scheduler.scheduler import RemoteTailPhase -from tensorrt_llm._torch.speculative import ( - SpecMetadata, - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, -) +from tensorrt_llm._torch.speculative import SpecMetadata from tensorrt_llm._torch.speculative.utils import get_static_draft_len from tensorrt_llm._torch.utils import get_per_request_prefill_cuda_graph_flag from tensorrt_llm.inputs.multimodal import _has_mm_payload_keys @@ -505,9 +501,7 @@ def run( postprocess_inputs: Callable[[dict[str, Any]], None], *, attn_metadata: AttentionMetadata, - draft_kv_cache_manager: KVCacheManager | KVCacheManagerV2 | None, moe_load_balancer: MoeLoadBalancer | None, - enable_spec_decode: bool, gather_context_logits: bool, position_id_offsets: torch.Tensor | None, static_inputs_staged: int = 0, @@ -528,9 +522,7 @@ def run( run_model, postprocess_inputs, attn_metadata=attn_metadata, - draft_kv_cache_manager=draft_kv_cache_manager, moe_load_balancer=moe_load_balancer, - enable_spec_decode=enable_spec_decode, gather_context_logits=gather_context_logits, position_id_offsets=position_id_offsets, static_inputs_staged=static_inputs_staged, @@ -544,9 +536,7 @@ def _run( postprocess_inputs: Callable[[dict[str, Any]], None], *, attn_metadata: AttentionMetadata, - draft_kv_cache_manager: KVCacheManager | KVCacheManagerV2 | None, moe_load_balancer: MoeLoadBalancer | None, - enable_spec_decode: bool, gather_context_logits: bool, position_id_offsets: torch.Tensor | None, static_inputs_staged: int = 0, @@ -577,41 +567,27 @@ def capture_model(model_inputs: dict[str, Any]) -> Any: key, capture_model, inputs, - enable_spec_decode=enable_spec_decode, postprocess_fn=postprocess_inputs, + prepare_each_forward=True, ) if self.cuda_graph_runner.is_warmup_only: outputs = capture_outputs elif needs_capture: - # Refresh attention metadata for the current batch's - # draft cache before replaying the captured graph. - saved_draft = prepare_attn_metadata_for_draft_replay( - attn_metadata, draft_kv_cache_manager + outputs = self.cuda_graph_runner.replay( + key, + inputs, + position_id_offsets=position_id_offsets, + static_inputs_staged=static_inputs_staged, ) - try: + else: + with MoeLoadBalancerIterContext(moe_load_balancer): outputs = self.cuda_graph_runner.replay( key, inputs, position_id_offsets=position_id_offsets, static_inputs_staged=static_inputs_staged, ) - finally: - restore_attn_metadata_after_draft_replay(attn_metadata, saved_draft) - else: - saved_draft = prepare_attn_metadata_for_draft_replay( - attn_metadata, draft_kv_cache_manager - ) - try: - with MoeLoadBalancerIterContext(moe_load_balancer): - outputs = self.cuda_graph_runner.replay( - key, - inputs, - position_id_offsets=position_id_offsets, - static_inputs_staged=static_inputs_staged, - ) - finally: - restore_attn_metadata_after_draft_replay(attn_metadata, saved_draft) return outputs def _get_graphs_to_capture( diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py index df05cc46e71e..3830e814da03 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py @@ -11,8 +11,8 @@ import torch from tensorrt_llm._device_copy import DeviceCopyBuffer -from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.features import HelixMetadata +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata, BatchInputs from tensorrt_llm._torch.distributed import Distributed from tensorrt_llm._torch.kernels.spec_step_copies import op as spec_step_copies from tensorrt_llm._torch.metadata import KVCacheParams @@ -158,6 +158,9 @@ class DecoderInputBuffers(InputBuffers): previous_pos_indices_cuda: torch.Tensor | None = None previous_pos_id_offsets_cuda: torch.Tensor | None = None previous_kv_lens_offsets_cuda: torch.Tensor | None = None + # previous_kv_lens_offsets_cuda laid out per sequence of the step, the + # step's BatchInputs.kv_lens_offsets. + kv_lens_offsets_cuda: torch.Tensor | None = None mrope_position_ids_cuda: torch.Tensor | None = None @classmethod @@ -207,6 +210,11 @@ def for_config( if spec else None ), + kv_lens_offsets_cuda=( + torch.zeros((config.max_batch_size,), dtype=torch.int, device="cuda") + if spec + else None + ), mrope_position_ids_cuda=( torch.empty((3, 1, config.max_num_tokens), dtype=torch.int, device="cuda") if config.use_mrope @@ -324,7 +332,7 @@ class HostInputs: previous_batch_tokens_per_request: list[int] | None = None host_position_ids: torch.Tensor | None = None num_cached_tokens_snapshot: list[int] | None = None - remote_tail_modes: set[Any] = field(default_factory=set) + has_remote_tail_context: bool = False spec_all_rank_counts: list[list[int]] | None = None virtual_num_tokens: int = 0 @@ -751,23 +759,20 @@ def _apply_steady_gen_fast_prepare( ), ) - if not attn_metadata.is_cuda_graph: - attn_metadata.seq_lens = cache["seq_lens_ones"] - attn_metadata.beam_width = 1 - attn_metadata.request_ids = cache["request_ids"] - attn_metadata.prompt_lens = cache["prompt_lens"] - attn_metadata.num_contexts = 0 - attn_metadata.num_chunked_ctx_requests = 0 - attn_metadata.kv_cache_params = KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=num_cached_tokens_per_seq, - num_extra_kv_tokens=get_num_extra_kv_tokens(None), - ) - attn_metadata.kv_cache_manager = kv_cache_manager - if hasattr(self.model.model_config.pretrained_config, "chunk_size"): - attn_metadata.mamba_chunk_size = self.model.model_config.pretrained_config.chunk_size with nvtx_range("steady_gen_metadata_prepare"): - attn_metadata.prepare() + attn_metadata.prepare( + BatchInputs( + seq_lens=None if attn_metadata.is_cuda_graph else cache["seq_lens_ones"], + num_contexts=0, + request_ids=cache["request_ids"], + prompt_lens=cache["prompt_lens"], + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq, + num_extra_kv_tokens=get_num_extra_kv_tokens(None), + ), + ) + ) virtual_num_tokens = self.pad_tokens(attn_metadata, num_requests, 0) @@ -879,22 +884,25 @@ def _apply_steady_mtp_fast_prepare( requests, new_tensors_device, self._buffers, attn_metadata ) - attn_metadata.beam_width = 1 - attn_metadata.request_ids = record.request_ids - attn_metadata.prompt_lens = record.prompt_lengths - attn_metadata.num_contexts = 0 - attn_metadata.num_chunked_ctx_requests = 0 - attn_metadata.kv_cache_params = KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=num_cached_tokens_per_seq, - num_extra_kv_tokens=record.num_extra_kv_tokens, - use_full_generation_page_table=record.use_full_generation_page_table, + batch_inputs = BatchInputs( + seq_lens=None, + num_contexts=0, + request_ids=record.request_ids, + prompt_lens=record.prompt_lengths, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq, + num_extra_kv_tokens=record.num_extra_kv_tokens, + use_full_generation_page_table=record.use_full_generation_page_table, + ), + kv_lens_offsets=self._overlap_kv_lens_offsets( + attn_metadata, kv_cache_manager, 0, 0, enable_spec_decode=True + ), ) - attn_metadata.kv_cache_manager = kv_cache_manager - if hasattr(self.model.model_config.pretrained_config, "chunk_size"): - attn_metadata.mamba_chunk_size = self.model.model_config.pretrained_config.chunk_size + if batch_inputs.kv_lens_offsets is not None: + batch_inputs.position_id_offsets = self._buffers.previous_pos_id_offsets_cuda with nvtx_range("steady_mtp_metadata_prepare"): - steady_mtp.prepare_attention(attn_metadata, kv_lens) + steady_mtp.prepare_attention(attn_metadata, batch_inputs, kv_lens) virtual_num_tokens = self.pad_tokens(attn_metadata, num_tokens, 0) @@ -1128,6 +1136,7 @@ def prepare_inputs( spec_config, enable_spec_decode=enable_spec_decode, is_dummy=is_dummy, + runtime_tokens_per_gen_step=runtime_tokens_per_gen_step, ) extra_model_inputs = host.extra_inputs.build(attn_metadata, resource_manager) @@ -2439,6 +2448,7 @@ def _prepare_attn_metadata( *, enable_spec_decode: bool, is_dummy: bool, + runtime_tokens_per_gen_step: int, ) -> None: num_cached_tokens_per_seq = host.lists.num_cached_tokens_per_seq gen_request_seq_slots = host.lists.gen_request_seq_slots @@ -2448,18 +2458,20 @@ def _prepare_attn_metadata( # A non-None owned-count list is what arms # _helix_spec_tokens_valid, and the per-token slots/bounds that # flag gates are only ever filled by recompute_helix_spec_buffers, - # which preprocess_inputs runs under enable_spec_decode. Gate the + # which Helix's prepare runs under enable_spec_decode. Gate the # hand-off here, at the single choke point, so no packing loop can # arm the spec path for ordinary helix generation and send its # consumers to uninitialized buffers. helix_spec_active = bool(enable_spec_decode and host.lists.helix_owned_new_tokens) - attn_metadata.update_helix_param( - helix_position_offsets=host.lists.helix_position_offsets, - helix_is_inactive_rank=host.lists.helix_is_inactive_rank, - helix_owned_new_tokens=( - host.lists.helix_owned_new_tokens if helix_spec_active else None - ), - ) + if attn_metadata.find_ext(HelixMetadata) is not None: + attn_metadata.update_helix_param( + helix_position_offsets=host.lists.helix_position_offsets, + helix_is_inactive_rank=host.lists.helix_is_inactive_rank, + helix_owned_new_tokens=( + host.lists.helix_owned_new_tokens if helix_spec_active else None + ), + tokens_per_gen_seq=runtime_tokens_per_gen_step, + ) if self._dspark_trims_submitted_tokens: self._publish_gen_token_layout( @@ -2467,6 +2479,7 @@ def _prepare_attn_metadata( scheduled_requests.generation_requests, runtime_draft_len=host.runtime_draft_len, ) + seq_lens = None refresh_seq_lens = not attn_metadata.is_cuda_graph if ( not refresh_seq_lens @@ -2474,19 +2487,24 @@ def _prepare_attn_metadata( and attn_metadata.seq_lens is not None and len(host.lists.sequence_lengths) == attn_metadata.seq_lens.shape[0] ): + # A ragged verify step changes the generation lengths of a CUDA + # graph metadata; they are copied into its bound buffer. refresh_seq_lens = True if refresh_seq_lens: # Assumes seq lens do not change between CUDA graph invocations. This applies # to draft sequences too. This means that all draft sequences must be padded. - attn_metadata.seq_lens = torch.tensor( + seq_lens = torch.tensor( host.lists.sequence_lengths, dtype=torch.int, pin_memory=prefer_pinned(), ) + beam_width = 1 num_generation_requests = len(gen_request_seq_slots) # Cache indirection is only used for beam search on generation requests if self.use_beam_search and num_generation_requests > 0: + # Without a cache indirection buffer the bound width stays. + beam_width = attn_metadata.beam_width if cache_indirection_buffer is not None: # Copy cache indirection to local buffer with offsets changing: seq_slots[i] -> i # Convert to GPU tensor to avoid implicit sync @@ -2502,34 +2520,12 @@ def _prepare_attn_metadata( out=self._buffers.cache_indirection[:num_generation_requests], ) if cache_indirection_buffer is not None or is_dummy: - attn_metadata.beam_width = self._config.max_beam_width - else: - attn_metadata.beam_width = 1 + beam_width = self._config.max_beam_width - attn_metadata.request_ids = host.lists.request_ids - attn_metadata.prompt_lens = host.lists.prompt_lengths - attn_metadata.num_contexts = scheduled_requests.num_context_requests - remote_tail_modes = { - getattr(request, "py_csa2_remote_tail_mode", None) - for request in scheduled_requests.context_requests - } - remote_tail_modes.discard(None) - if remote_tail_modes: - if len(remote_tail_modes) != 1 or any( - getattr(request, "py_csa2_remote_tail_mode", None) is None - for request in scheduled_requests.context_requests - ): - raise ValueError("CSA2 remote-tail context batches must be homogeneous") - attn_metadata.csa2_remote_tail_mode = next(iter(remote_tail_modes)) - attn_metadata.csa2_remote_tail_starts = [ - request.py_csa2_remote_tail_start for request in scheduled_requests.context_requests - ] - else: - attn_metadata.csa2_remote_tail_mode = None - attn_metadata.csa2_remote_tail_starts = [] + num_contexts = scheduled_requests.num_context_requests # Use num_chunked_ctx_requests to record the number of extend context requests, - # so that we can update the kv_lens_cuda correctly in preprocess_inputs. - attn_metadata.num_chunked_ctx_requests = 0 + # so that the overlap scheduler's KV-length correction reaches their rows. + num_chunked_ctx_requests = 0 if ( enable_spec_decode and spec_config.spec_dec_mode.extend_ctx(self._config.attention_backend) @@ -2537,38 +2533,92 @@ def _prepare_attn_metadata( ): # For the tree decoding, we want to use XQA to process the draft tokens for the target model. # Therefore, we do not treat them as the chunked context requests. - attn_metadata.num_contexts += len(extend_requests) - attn_metadata.num_chunked_ctx_requests = len(extend_requests) - - attn_metadata.kv_cache_params = KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=num_cached_tokens_per_seq, - num_extra_kv_tokens=get_num_extra_kv_tokens(spec_config), - use_full_generation_page_table=( - self._should_use_full_generation_page_table(spec_config, attn_metadata) - ), - ) - attn_metadata.kv_cache_manager = kv_cache_manager + num_contexts += len(extend_requests) + num_chunked_ctx_requests = len(extend_requests) - if hasattr(self.model.model_config.pretrained_config, "chunk_size"): - attn_metadata.mamba_chunk_size = self.model.model_config.pretrained_config.chunk_size # Some sparse backends (RocketKV) clamp # kv_cache_params.num_cached_tokens_per_seq in place during prepare(), # and KVCacheParams holds the list by reference. Snapshot the true # pre-prepare counts so the steady-gen recording below stores values # that the per-step prepare() can re-clamp from scratch. num_cached_tokens_snapshot = list(num_cached_tokens_per_seq) - prepare_context_replay = getattr(attn_metadata, "prepare_context_replay", None) - if prepare_context_replay is not None: - prepare_context_replay( - scheduled_requests.context_requests, - allow_final_window=( - self._config.prefill_cuda_graph_backend == PrefillCudaGraphBackend.DISABLED + kv_lens_offsets = self._overlap_kv_lens_offsets( + attn_metadata, + kv_cache_manager, + num_contexts, + num_chunked_ctx_requests, + enable_spec_decode=enable_spec_decode, + ) + attn_metadata.prepare( + BatchInputs( + seq_lens=seq_lens, + num_contexts=num_contexts, + request_ids=host.lists.request_ids, + prompt_lens=host.lists.prompt_lengths, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq, + num_extra_kv_tokens=get_num_extra_kv_tokens(spec_config), + use_full_generation_page_table=( + self._should_use_full_generation_page_table(spec_config, attn_metadata) + ), ), - ) - attn_metadata.prepare() + beam_width=beam_width, + num_chunked_ctx_requests=num_chunked_ctx_requests, + kv_lens_offsets=kv_lens_offsets, + position_id_offsets=( + None if kv_lens_offsets is None else self._buffers.previous_pos_id_offsets_cuda + ), + ), + context_requests=scheduled_requests.context_requests, + prefill_cuda_graphs=( + self._config.prefill_cuda_graph_backend != PrefillCudaGraphBackend.DISABLED + ), + ) host.num_cached_tokens_snapshot = num_cached_tokens_snapshot - host.remote_tail_modes = remote_tail_modes + host.has_remote_tail_context = any( + getattr(request, "py_csa2_remote_tail_mode", None) is not None + for request in scheduled_requests.context_requests + ) + + def _overlap_kv_lens_offsets( + self, + attn_metadata: AttentionMetadata, + kv_cache_manager: KVCacheManager | KVCacheManagerV2 | None, + num_contexts: int, + num_chunked_ctx_requests: int, + *, + enable_spec_decode: bool, + ) -> torch.Tensor | None: + """The step's per-sequence KV-length corrections of the overlap scheduler. + + With speculative decoding the overlap scheduler schedules each + generation request as if every draft token of its previous step were + accepted; the accepted counts are known only on device + (``previous_kv_lens_offsets_cuda``, one row per generation request, or + per extend request when those run as chunked context). They are laid + out per sequence for the metadata's ``prepare`` to apply; None when no + correction applies. + """ + if ( + not enable_spec_decode + or self._config.disable_overlap_scheduler + or kv_cache_manager is None + or not attn_metadata.supports_kv_lens_offsets + ): + return None + offsets = self._buffers.kv_lens_offsets_cuda + previous = self._buffers.previous_kv_lens_offsets_cuda + offsets.zero_() + if num_contexts >= num_chunked_ctx_requests > 0: + # Extend requests run as the trailing context requests. + first_row = num_contexts - num_chunked_ctx_requests + num_rows = num_chunked_ctx_requests + else: + first_row = num_contexts + num_rows = offsets.shape[0] - num_contexts + offsets[first_row : first_row + num_rows].copy_(previous[:num_rows]) + return offsets def _pad_inputs( self, @@ -2626,7 +2676,7 @@ def _pad_inputs( host.num_ctx_requests, attn_all_rank_num_tokens, allow_prefill_cuda_graph=( - not host.remote_tail_modes + not host.has_remote_tail_context and scheduled_requests.attention_dp_phase is not RemoteTailPhase.CONTEXT ), ) @@ -2913,27 +2963,12 @@ def _get_remote_tail_decode_rank_tokens( ) return gathered[:, 0].tolist() - def _kv_lens_corrected_in_preprocess( - self, inputs: dict[str, Any], enable_spec_decode: bool - ) -> bool: - """Whether preprocess_inputs applies the overlap scheduler's KV-length offsets to kv_lens_cuda and then calls - on_update_kv_lens (the branch below).""" - attn_meta = inputs.get("attn_metadata") - return ( - enable_spec_decode - and not self._config.disable_overlap_scheduler - and attn_meta is not None - and attn_meta.kv_cache_manager is not None - and hasattr(attn_meta, "kv_lens_cuda") - ) - def preprocess_inputs( self, inputs: dict[str, Any], *, enable_spec_decode: bool, runtime_draft_len: int ) -> dict[str, Any]: """ Make some changes to the device inputs and avoid blocking the async data transfer """ - attn_meta = inputs.get("attn_metadata") # A model's decode prologue may replace the device work below with one # launch; it decides on the host before any device work and returns # False to keep the code below. @@ -2944,15 +2979,6 @@ def preprocess_inputs( if self.guided_decoder is not None: self.guided_decoder.token_event.record() return inputs - # Overlap scheduling corrects kv_lens_cuda below from the runtime - # accepted-token counts and calls on_update_kv_lens() again after it. - has_kv_lens_correction = self._kv_lens_corrected_in_preprocess(inputs, enable_spec_decode) - # Invalidate per-forward-pass caches so they are recomputed (and captured) on every _forward_step. - # With a correction, the call after it does this: every on_update_kv_lens rebuilds - # from the current lengths and nothing reads its outputs in between. - if attn_meta is not None and not has_kv_lens_correction: - attn_meta.on_update_kv_lens() - if enable_spec_decode and not self._config.disable_overlap_scheduler: # When enabling overlap scheduler, the kv cache for draft tokens will # be prepared in advance by using the max_total_draft_tokens. But we need to use @@ -2961,31 +2987,11 @@ def preprocess_inputs( # we need to preprocess the inputs in forward to update the position_ids and # kv cache length. if inputs["attn_metadata"].kv_cache_manager is not None: - num_seqs = inputs["attn_metadata"].num_seqs - num_ctx_requests = inputs["attn_metadata"].num_contexts num_gen_requests = inputs["attn_metadata"].num_generations num_ctx_tokens = inputs["attn_metadata"].num_ctx_tokens num_chunked_ctx_requests = inputs["attn_metadata"].num_chunked_ctx_requests previous_batch_tokens = inputs["input_ids"].shape[0] - num_ctx_tokens - # RuntimeHooks.spec_offsets_op: both corrections of a generation-row step in one launch. - fused_offsets = ( - self.runtime_hooks.spec_offsets_op is not None - and inputs["position_ids"].ndim != 3 - and hasattr(inputs["attn_metadata"], "kv_lens_cuda") - and not ( - num_ctx_requests >= num_chunked_ctx_requests - and num_chunked_ctx_requests > 0 - ) - and self.runtime_hooks.spec_offsets_op( - inputs["position_ids"][0, num_ctx_tokens:], - self._buffers.previous_pos_id_offsets_cuda[:previous_batch_tokens], - inputs["attn_metadata"].kv_lens_cuda[num_ctx_requests:num_seqs], - self._buffers.previous_kv_lens_offsets_cuda[:num_gen_requests], - ) - ) - if fused_offsets: - pass - elif inputs["position_ids"].ndim == 3: # mrope: [3, 1, N] + if inputs["position_ids"].ndim == 3: # mrope: [3, 1, N] inputs["position_ids"][:, :, num_ctx_tokens:] += ( self._buffers.previous_pos_id_offsets_cuda[:previous_batch_tokens] ) @@ -2994,26 +3000,12 @@ def preprocess_inputs( self._buffers.previous_pos_id_offsets_cuda[:previous_batch_tokens] ) - if has_kv_lens_correction: - if fused_offsets: - pass - elif ( - num_ctx_requests >= num_chunked_ctx_requests - and num_chunked_ctx_requests > 0 - ): - # The generation requests with draft_tokens are treated as chunked context - # requests when extend_ctx returns True. - inputs["attn_metadata"].kv_lens_cuda[ - num_ctx_requests - num_chunked_ctx_requests : num_ctx_requests - ] += self._buffers.previous_kv_lens_offsets_cuda[:num_chunked_ctx_requests] - else: - inputs["attn_metadata"].kv_lens_cuda[num_ctx_requests:num_seqs] += ( - self._buffers.previous_kv_lens_offsets_cuda[:num_gen_requests] - ) - inputs["attn_metadata"].on_update_kv_lens() - # TRTLLM uses `kv_lens_cuda` above; FlashInfer exposes this backend-specific - # correction without coupling the engine to its metadata type. - elif hasattr(inputs["attn_metadata"], "apply_spec_decode_kv_lens_offsets"): + # Metadata that supports kv_lens_offsets applied the KV-length + # correction in prepare; FlashInfer applies it here, through + # its own hook. + if not inputs["attn_metadata"].supports_kv_lens_offsets and hasattr( + inputs["attn_metadata"], "apply_spec_decode_kv_lens_offsets" + ): inputs["attn_metadata"].apply_spec_decode_kv_lens_offsets( self._buffers.previous_kv_lens_offsets_cuda, num_gen_requests, @@ -3021,33 +3013,6 @@ def preprocess_inputs( num_chunked_contexts=num_chunked_ctx_requests, ) - if enable_spec_decode and self.mapping.has_cp_helix(): - # Helix verify groups: the per-token device buffers (write slots, - # attention bounds, rank-local kv lens) must be derived on EVERY - # spec step, overlap or not -- the append/mask kernels consume - # them whenever _helix_spec_tokens_valid is armed. Under overlap - # the host packed provisional positions from a stale base, so - # first apply the same accepted-count correction position_ids - # got above; without overlap the host values are already exact. - md = inputs.get("attn_metadata") - if ( - md is not None - and md.kv_cache_manager is not None - and getattr(md, "_helix_spec_tokens_valid", False) - ): - helix_gen_tokens = inputs["input_ids"].shape[0] - md.num_ctx_tokens - if not self._config.disable_overlap_scheduler: - # The kv_lens override in the recompute supersedes the - # generic previous_kv_lens_offsets adjustment above, - # which is not ownership-aware. - md.helix_position_offsets[:helix_gen_tokens] += ( - self._buffers.previous_pos_id_offsets_cuda[:helix_gen_tokens] - ) - md.recompute_helix_spec_buffers( - helix_gen_tokens, self.get_runtime_tokens_per_gen_step(runtime_draft_len) - ) - md.on_update_kv_lens() - if self.guided_decoder is not None: self.guided_decoder.token_event.record() @@ -3063,8 +3028,6 @@ def postprocess_inputs( """ if enable_spec_decode and not self._config.disable_overlap_scheduler: if inputs["attn_metadata"].kv_cache_manager is not None: - num_seqs = inputs["attn_metadata"].num_seqs - num_ctx_requests = inputs["attn_metadata"].num_contexts num_gen_requests = inputs["attn_metadata"].num_generations num_ctx_tokens = inputs["attn_metadata"].num_ctx_tokens num_chunked_ctx_requests = inputs["attn_metadata"].num_chunked_ctx_requests @@ -3078,22 +3041,11 @@ def postprocess_inputs( self._buffers.previous_pos_id_offsets_cuda[:previous_batch_tokens] ) - # Only TrtllmAttentionMetadata has kv_lens_cuda. - if isinstance(inputs["attn_metadata"], TrtllmAttentionMetadata): - if ( - num_ctx_requests >= num_chunked_ctx_requests - and num_chunked_ctx_requests > 0 - ): - inputs["attn_metadata"].kv_lens_cuda[ - num_ctx_requests - num_chunked_ctx_requests : num_ctx_requests - ] -= self._buffers.previous_kv_lens_offsets_cuda[:num_chunked_ctx_requests] - else: - inputs["attn_metadata"].kv_lens_cuda[num_ctx_requests:num_seqs] -= ( - self._buffers.previous_kv_lens_offsets_cuda[:num_gen_requests] - ) # Restore the FlashInfer-specific logical KV lengths through the same optional hook # used by `preprocess_inputs`. - elif hasattr(inputs["attn_metadata"], "apply_spec_decode_kv_lens_offsets"): + if not inputs["attn_metadata"].supports_kv_lens_offsets and hasattr( + inputs["attn_metadata"], "apply_spec_decode_kv_lens_offsets" + ): inputs["attn_metadata"].apply_spec_decode_kv_lens_offsets( self._buffers.previous_kv_lens_offsets_cuda, num_gen_requests, @@ -3102,18 +3054,6 @@ def postprocess_inputs( restore=True, ) - if self.mapping.has_cp_helix() and getattr( - inputs["attn_metadata"], "_helix_spec_tokens_valid", False - ): - # Mirror of the helix position correction in - # preprocess_inputs (capture symmetry, like position_ids - # above). The recompute's OVERWRITES (slots/bounds/ - # kv_lens) need no reversal: every consumer buffer is - # rewritten from host state at the next step's prepare. - inputs["attn_metadata"].helix_position_offsets[:previous_batch_tokens] -= ( - self._buffers.previous_pos_id_offsets_cuda[:previous_batch_tokens] - ) - def ragged_verify_token_buckets(self, padded_bs: int) -> list[int]: """Measured compact ``V`` cells captured for one exact ``G``.""" if self._config.spec_config is None: @@ -3446,18 +3386,22 @@ def _apply_device_window_prologue(self, inputs, new_tensors_device) -> bool: # (w - S): the single-delta variant left every re-ranked request's # kv_len off by (w - S), which shifted the indexer K-cache slot # mapping (slot_mapping_fp8) and silently wrote K entries into the - # wrong cache slots. Established empirically by an A/B tensor diff - # against a full host restage with the same windows. + # wrong cache slots. + # When prepare applied this step's KV-length offsets (overlap + # scheduler: new_tokens_lens - S per request), the true windows make + # them new_tokens_lens - w, i.e. -(w - S) each. Combined: (past + 2S) + # + (new - S) + 2(w-S) - (w-S) = past + w + new. The offset buffers + # move by -(w - S) as well, so they describe the true windows to any + # later reader. window_delta = result.verify_lens - split_lens - attn_metadata.kv_lens_cuda[:padded_bs] += 2 * window_delta - # The graph adds previous_kv_lens_offsets (staged as new_tokens_lens - - # shape_lens, per request) to kv_lens during replay; host-with-w - # stages new_tokens_lens - w there, so the offsets move by -(w - S). - # Combined: (past + 2S) + 2(w-S) + (new - S) - (w-S) = past + w + new, - # exactly the host-with-w in-graph sum. - self._buffers.previous_kv_lens_offsets_cuda[:padded_bs] -= window_delta.to( - self._buffers.previous_kv_lens_offsets_cuda.dtype - ) + offsets_applied = attn_metadata.kv_lens_offsets is not None + kv_lens_delta = window_delta if offsets_applied else 2 * window_delta + for offsets in ( + self._buffers.previous_kv_lens_offsets_cuda, + self._buffers.kv_lens_offsets_cuda, + ): + if offsets is not None: + offsets[:padded_bs] -= window_delta.to(offsets.dtype) req_idx = result.req_idx spec_metadata.remap_expanded_sampling_params(req_idx, bucket) @@ -3503,7 +3447,7 @@ def _apply_device_window_prologue(self, inputs, new_tensors_device) -> bool: ).to(self._buffers.draft_tokens_cuda.dtype) ) - apply_device_layout(result.verify_lens, req_idx, result.kv_correction) + apply_device_layout(result.verify_lens, req_idx, result.kv_correction, kv_lens_delta) return True @staticmethod diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py index bdb21380d0c4..b018fce34166 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py @@ -9,10 +9,11 @@ import torch +from tensorrt_llm._torch.attention.backends.features import RecurrentConfig +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 -from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import BaseMambaCacheManager from tensorrt_llm._torch.pyexecutor.resource_manager import ( BaseResourceManager, KVCacheManager, @@ -88,6 +89,7 @@ def setup_attention( self, kv_cache_manager: KVCacheManager | KVCacheManagerV2, draft_kv_cache_manager: KVCacheManager | KVCacheManagerV2 | None = None, + spec_decoding: SpecDecodingConfig | None = None, ) -> AttentionMetadata: if self.attn_metadata is not None: # This assertion can be relaxed if needed: just create a new metadata @@ -110,16 +112,30 @@ def setup_attention( kv_cache_manager=kv_cache_manager, draft_kv_cache_manager=draft_kv_cache_manager, enable_ragged_verification=self._ragged_verification_enabled, + recurrent_config=RecurrentConfig( + chunk_size=getattr(config, "chunk_size", RecurrentConfig.chunk_size), + mamba_metadata_cls=resolve_mamba_metadata_cls(self.model), + ), + spec_decoding=spec_decoding, + one_model_draft=( + self._config.spec_config is not None + and self._config.spec_config.spec_dec_mode.use_one_engine() + ), + model_extension_types=self.runtime_hooks.metadata_extensions, + model_extension_config=self._model_metadata_config(), ) - if isinstance(kv_cache_manager, BaseMambaCacheManager): - self.attn_metadata.mamba_chunk_size = getattr( - config, "chunk_size", self.attn_metadata.mamba_chunk_size - ) - self.attn_metadata.mamba_metadata_cls = resolve_mamba_metadata_cls(self.model) self._arm_msa_policy(kv_cache_manager) return self.attn_metadata + def _model_metadata_config(self) -> dict: + """The creation arguments the model gives its metadata's extensions.""" + config = dict(self.runtime_hooks.metadata_config) + if self.runtime_hooks.msa != MsaRuntimePolicy(): + # The MiniMax-M3 MSA metadata takes the model's preparation choices. + config["msa_policy"] = self.runtime_hooks.msa + return config + def _arm_msa_policy(self, kv_cache_manager: KVCacheManager | KVCacheManagerV2 | None) -> None: """Hand the model's MiniMax-M3 MSA preparation choices to the metadata and the KV cache.""" msa = self.runtime_hooks.msa @@ -140,7 +156,8 @@ def _arm_msa_policy(self, kv_cache_manager: KVCacheManager | KVCacheManagerV2 | "RuntimeHooks.msa requires the MiniMax-M3 MSA sparse attention backend and KV cache, " f"got {type(self.attn_metadata).__name__} and {type(kv_cache_manager).__name__}" ) - self.attn_metadata.msa_policy = msa + # The policy reached the MSA metadata as creation configuration + # (msa_policy); the KV cache takes the decode shortcuts it allows. kv_cache_manager.configure_decode_shortcuts( skip_unchanged_block_offsets=msa.skip_unchanged_block_offsets, reuse_pool_views=msa.reuse_pool_views, @@ -192,7 +209,13 @@ def _warmup_dspark_ragged_compressor_metadata(self) -> None: if not self._ragged_verification_enabled: return - metadata = getattr(self, "attn_metadata", None) + from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( + DeepseekV4Metadata, + ) + + attn_metadata = getattr(self, "attn_metadata", None) + # The DeepSeek-V4 extension owns the compressor buffers and helpers. + metadata = None if attn_metadata is None else attn_metadata.find_ext(DeepseekV4Metadata) required_attributes = ( "_compute_compressed_mask", "_compute_gen_compressed_position_ids", @@ -352,9 +375,28 @@ def update_spec_metadata( runtime_draft_len ) - is_spec_dec_mode = spec_metadata.spec_dec_mode.attention_need_spec_dec_mode( - config.attention_backend + if spec_tree_manager is not None: + spec_tree_manager.slot_storage.fill_all_slot_ids( + scheduled_requests.context_requests, + scheduled_requests.generation_requests, + ) + + attn_metadata.update_spec_dec_param( + batch_size=scheduled_requests.batch_size, + spec_metadata=spec_metadata, + spec_tree_manager=spec_tree_manager, + num_contexts=scheduled_requests.num_context_requests, ) + + +def spec_decoding_config( + config: DecoderRunnerConfig, + spec_metadata: SpecMetadata | None, + spec_tree_manager: SpecTreeManager | None, +) -> SpecDecodingConfig | None: + """The speculative-decoding settings the attention metadata is created with.""" + if spec_metadata is None: + return None # Parallel-draft modes advertise their full generation width rather than a # conventional draft length, so attention needs the total-token capacity. if spec_metadata.spec_dec_mode.is_parallel_draft(): @@ -363,23 +405,17 @@ def update_spec_metadata( else: max_draft_len = config.original_max_draft_len max_total_draft_tokens = config.spec_dec_max_total_draft_tokens - - if spec_tree_manager is not None: - spec_tree_manager.slot_storage.fill_all_slot_ids( - scheduled_requests.context_requests, - scheduled_requests.generation_requests, - ) - - attn_metadata.update_spec_dec_param( - batch_size=scheduled_requests.batch_size, - is_spec_decoding_enabled=is_spec_dec_mode, + return SpecDecodingConfig( + attention_needs_spec_dec_mode=spec_metadata.spec_dec_mode.attention_need_spec_dec_mode( + config.attention_backend + ), is_spec_dec_tree=spec_metadata.is_spec_dec_tree, is_spec_dec_dynamic_tree=spec_metadata.is_spec_dec_dynamic_tree, max_draft_len=max_draft_len, max_total_draft_tokens=max_total_draft_tokens, - spec_metadata=spec_metadata, - spec_tree_manager=spec_tree_manager, - num_contexts=scheduled_requests.num_context_requests, + tree_buffer_dim=( + spec_tree_manager._internal_buf_dim if spec_tree_manager is not None else 0 + ), ) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/runner.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/runner.py index 7419e578f422..d2725bdf6f82 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/runner.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/runner.py @@ -34,7 +34,6 @@ CUDAGraphRunner, CUDAGraphRunnerConfig, DSparkCUDAGraphRunner, - KeyType, ) from tensorrt_llm._torch.pyexecutor.guided_decoder import CapturableGuidedDecoder from tensorrt_llm._torch.pyexecutor.h2d_cache import H2DSkipCache @@ -84,6 +83,7 @@ from .metadata import ( DecoderMetadata, get_spec_managers, + spec_decoding_config, sync_group_all_greedy_sample, update_spec_metadata, validate_dspark_draft_sampling, @@ -163,7 +163,6 @@ class DecoderRunner(ScheduledModelRunner): _context_graph_promotion_supported = True _steady_gen_cache_supported = True _eager_workspace_reclaim_supported = True - _deferred_decode_outputs_supported = True def __init__( self, @@ -635,12 +634,18 @@ def _forward_decoder( self._config.spec_config, resource_manager ) - attn_metadata = self._metadata.setup_attention(kv_cache_manager, draft_kv_cache_manager) + spec_resource_manager, spec_tree_manager = get_spec_managers(resource_manager) + if self._config.spec_config is not None: + self._metadata.setup_speculative(spec_resource_manager) + attn_metadata = self._metadata.setup_attention( + kv_cache_manager, + draft_kv_cache_manager, + spec_decoding_config(self._config, self._metadata.spec_metadata, spec_tree_manager), + ) if isinstance(attn_metadata, TrtllmAttentionMetadata): attn_metadata.trtllm_gen_jit_warmup = self._trtllm_gen_jit_warmup if enable_spec_decode: - spec_resource_manager, spec_tree_manager = get_spec_managers(resource_manager) - spec_metadata = self._metadata.setup_speculative(spec_resource_manager) + spec_metadata = self._metadata.spec_metadata assert spec_metadata is not None update_spec_metadata( spec_metadata, @@ -772,29 +777,22 @@ def _forward_decoder( and not self.cuda_graph_runner.needs_capture(key) ): step_inputs.begin() - with self._cuda_graph_metadata_prepare_scope( - attn_metadata, - key, + prepared = self._inputs.prepare_inputs( execution_requests, + kv_cache_manager, + attn_metadata, + spec_metadata, + new_tensors_device, + cache_indirection_buffer, + resource_manager, + can_run_graph, execution_promoted_context_ids, + use_lora_graph=use_lora_graph, enable_spec_decode=enable_spec_decode, - ): - prepared = self._inputs.prepare_inputs( - execution_requests, - kv_cache_manager, - attn_metadata, - spec_metadata, - new_tensors_device, - cache_indirection_buffer, - resource_manager, - can_run_graph, - execution_promoted_context_ids, - use_lora_graph=use_lora_graph, - enable_spec_decode=enable_spec_decode, - runtime_draft_len=runtime_draft_len, - is_dummy=is_dummy, - gather_context_logits=gather_context_logits, - ) + runtime_draft_len=runtime_draft_len, + is_dummy=is_dummy, + gather_context_logits=gather_context_logits, + ) inputs = prepared.kwargs runtime_draft_len = prepared.runtime_draft_len if execution_promoted_context_ids: @@ -839,9 +837,7 @@ def postprocess_inputs(model_inputs: dict[str, Any]) -> None: run_model, postprocess_inputs, attn_metadata=attn_metadata, - draft_kv_cache_manager=draft_kv_cache_manager, moe_load_balancer=moe_load_balancer, - enable_spec_decode=enable_spec_decode, gather_context_logits=gather_context_logits, static_inputs_staged=prepared.static_inputs_staged, position_id_offsets=( @@ -1163,39 +1159,6 @@ def _execute_logit_post_processors( ) logits_row_offset += row_stride - def _cuda_graph_metadata_prepare_scope( - self, - attn_metadata: AttentionMetadata, - key: KeyType | None, - scheduled_requests: ScheduledRequests, - promoted_context_request_ids: frozenset[int], - *, - enable_spec_decode: bool, - ) -> contextlib.AbstractContextManager[Any]: - """Allow derived outputs only when this forward owns a decode graph. - - Both capture/warmup and replay execute _preprocess_inputs before model - consumers. Non-speculative overlap has the same ordering; speculative - acceptance corrections and draft-manager swaps keep full preparation. - """ - defer = getattr(attn_metadata, "defer_cuda_graph_decode_outputs", None) - if ( - defer is not None - and isinstance(key, KeyType) - and attn_metadata.is_cuda_graph is True - and key.num_contexts == 0 - and key.draft_len == 0 - and scheduled_requests.num_context_requests == 0 - and scheduled_requests.num_generation_requests > 0 - and not promoted_context_request_ids - and not enable_spec_decode - and not self._use_beam_search - and self._deferred_decode_outputs_supported - and self._mapping.cp_size == 1 - ): - return defer() - return contextlib.nullcontext() - @staticmethod def _restore_attention_dp_semantic_outputs( scheduled_requests: ScheduledRequests, outputs: dict[str, Any] diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py index fbc4b957783d..496f38bf6a17 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/warmup.py @@ -44,7 +44,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.mapping import CpType, Mapping -from .metadata import resolve_mamba_metadata_cls +from .metadata import get_spec_managers, resolve_mamba_metadata_cls, spec_decoding_config if TYPE_CHECKING: from tensorrt_llm._torch.modules.embedding import LMHead @@ -597,7 +597,8 @@ def _freeze_eager_workspace_floor(self) -> EagerWorkspaceReclaimer | None: ) return None if ( - type(metadata) is not TrtllmAttentionMetadata + not isinstance(metadata, TrtllmAttentionMetadata) + or metadata.core_class() is not TrtllmAttentionMetadata or not metadata.workspace_reclaimable or metadata.workspace is None or metadata.workspace.untyped_storage().nbytes() == 0 @@ -650,11 +651,12 @@ def _warmup_dg_paged_mqa_logits_metadata(self) -> None: try: from tensorrt_llm._torch.attention.backends.sparse.dsa import ( _DG_SCHEDULE_BLOCK_KV, - DSAtrtllmAttentionMetadata, + DSAMetadata, ) except ImportError: return - if not isinstance(attn_meta, DSAtrtllmAttentionMetadata): + dsa = attn_meta.find_ext(DSAMetadata) + if dsa is None: return try: from tensorrt_llm.deep_gemm import get_paged_mqa_logits_metadata @@ -665,7 +667,7 @@ def _warmup_dg_paged_mqa_logits_metadata(self) -> None: ) return - num_sms = attn_meta.num_sms + num_sms = dsa.num_sms max_bs = max(1, int(self._config.max_batch_size)) beam_width = max(1, self._config.max_beam_width) # Static upper bound on the row-count multiplier applied to @@ -673,10 +675,8 @@ def _warmup_dg_paged_mqa_logits_metadata(self) -> None: # are bounded above by `(1 + max_draft_tokens)`; see the # docstring for why we don't read the runtime `dsl_expand_factor` # here. - max_draft_tokens = int(getattr(attn_meta, "max_draft_tokens", 0) or 0) - expands_batch = getattr(attn_meta, "use_expanded_buffers_for_mtp", False) or getattr( - attn_meta, "expand_for_dsl", False - ) + max_draft_tokens = int(dsa.max_draft_tokens or 0) + expands_batch = dsa.use_expanded_buffers_for_mtp or dsa.expand_for_dsl expand_factor = 1 + max_draft_tokens if expands_batch else 1 max_aligned = ((max_bs * beam_width * expand_factor + 31) // 32) * 32 buckets = list(range(32, max_aligned + 32, 32)) @@ -838,19 +838,25 @@ def _ensure_dsa_attn_metadata_for_warmup(self, resource_manager: ResourceManager if self._metadata.attn_metadata is not None: return try: - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAtrtllmAttentionMetadata + from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata except ImportError: return metadata_cls = getattr(self._config.attention_backend, "Metadata", None) - if metadata_cls is None or not issubclass(metadata_cls, DSAtrtllmAttentionMetadata): + extension_types = getattr(metadata_cls, "backend_extension_types", ()) + if not any(issubclass(cls, DSAMetadata) for cls in extension_types): return kv_cache_manager = resource_manager.get_resource_manager( ResourceManagerType.KV_CACHE_MANAGER ) if kv_cache_manager is None: return + spec_resource_manager, spec_tree_manager = get_spec_managers(resource_manager) + if self._config.spec_config is not None: + self._metadata.setup_speculative(spec_resource_manager) self._metadata.setup_attention( - kv_cache_manager, get_draft_kv_cache_manager(self._config.spec_config, resource_manager) + kv_cache_manager, + get_draft_kv_cache_manager(self._config.spec_config, resource_manager), + spec_decoding_config(self._config, self._metadata.spec_metadata, spec_tree_manager), ) def _warmup_cute_dsl_radix_topk(self) -> None: @@ -867,16 +873,14 @@ def _warmup_cute_dsl_radix_topk(self) -> None: if attn_meta is None: return try: - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAtrtllmAttentionMetadata + from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata except ImportError: return - if isinstance(attn_meta, DSAtrtllmAttentionMetadata): + dsa = attn_meta.find_ext(DSAMetadata) + if dsa is not None: next_n = 1 + self._config.original_max_draft_len - attn_meta.warmup_cute_dsl_radix_topk(next_n) - if hasattr(attn_meta, "warmup_selfsampling_topk"): - attn_meta.warmup_selfsampling_topk( - next_n, batch_sizes=self._config.cuda_graph_batch_sizes - ) + dsa.warmup_cute_dsl_radix_topk(next_n) + dsa.warmup_selfsampling_topk(next_n, batch_sizes=self._config.cuda_graph_batch_sizes) def _general_warmup( self, resource_manager: ResourceManager, warmup_requests_configs: list[tuple[int, int]] diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder.py index 031deb231975..1d4893409f15 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder.py @@ -20,6 +20,7 @@ AttentionBackend, AttentionMetadata, AttentionRuntimeFeatures, + BatchInputs, ) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.autotuner import AutoTuner, autotune @@ -328,9 +329,8 @@ def _create_attention_metadata( kv_cache_manager=None, enable_context_mla_with_cached_kv=enable_context_mla_with_cached_kv, max_num_heads_per_kv=max_num_heads_per_kv, + max_seq_len=self._config.max_seq_len, ) - metadata.block_ids_per_seq = None - metadata.kv_block_ids_per_seq = None return metadata def _is_distributed_forward(self) -> bool: @@ -683,17 +683,20 @@ def _prepare_encoder_inputs( ) metadata = self._setup_attention_metadata() - metadata.seq_lens = torch.tensor(sequence_lengths, dtype=torch.int) - metadata.num_contexts = batch_size - metadata.max_seq_len = self._config.max_seq_len - metadata.request_ids = list(range(batch_size)) if ( multi_item_part_lens is not None and not self._config.attention_backend.support_multi_item_scoring() ): raise ValueError("The selected attention backend does not support multi-item scoring.") - metadata.multi_item_part_lens = multi_item_part_lens - metadata.prepare_encoder_only() + metadata.prepare_encoder_only( + BatchInputs( + seq_lens=torch.tensor(sequence_lengths, dtype=torch.int), + num_contexts=batch_size, + request_ids=list(range(batch_size)), + prompt_lens=list(sequence_lengths), + multi_item_part_lens=multi_item_part_lens, + ) + ) self._buffers.input_ids_cuda[:actual_num_tokens].copy_(input_ids_cpu, non_blocking=True) self._buffers.position_ids_cuda[:actual_num_tokens].copy_( @@ -1173,10 +1176,6 @@ def _forward_step( *, gather_context_logits: bool, ) -> dict[str, Any]: - metadata = inputs.get("attn_metadata") - if metadata is not None: - metadata.on_update_kv_lens() - outputs = self._model_caller( **inputs, return_context_logits=gather_context_logits, diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder_decoder.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder_decoder.py index b1d178b729cc..f0aeb4165f7d 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder_decoder.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/encoder_decoder.py @@ -15,7 +15,9 @@ from tensorrt_llm._torch.attention.backends.interface import ( AttentionMetadata, AttentionRuntimeFeatures, + BatchInputs, ) +from tensorrt_llm._torch.attention.backends.sparse.params import SparseAlgorithmExtension from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata from tensorrt_llm._torch.distributed import Distributed @@ -100,13 +102,16 @@ def _build_attention_metadata( "Only vanilla and TRT-LLM attention metadata are supported " "for encoder-decoder encoder execution." ) - metadata.seq_lens = torch.tensor( - sequence_lengths, dtype=torch.int, pin_memory=prefer_pinned() + metadata.prepare_encoder_only( + BatchInputs( + seq_lens=torch.tensor( + sequence_lengths, dtype=torch.int, pin_memory=prefer_pinned() + ), + num_contexts=len(sequence_lengths), + request_ids=request_ids, + prompt_lens=list(sequence_lengths), + ) ) - metadata.num_contexts = len(sequence_lengths) - metadata.max_seq_len = self._config.max_seq_len - metadata.request_ids = request_ids - metadata.prepare_encoder_only() return metadata def _prepare_inputs( @@ -554,7 +559,9 @@ def _prepare_inputs_fast_path( ) -> PreparedInputs | None: if ( not promoted_context_request_ids - and type(attn_metadata) is TrtllmAttentionMetadata + and isinstance(attn_metadata, TrtllmAttentionMetadata) + and attn_metadata.core_class() is TrtllmAttentionMetadata + and attn_metadata.find_ext(SparseAlgorithmExtension) is None and self._can_use_fast_path( scheduled_requests, new_tokens_device, next_draft_tokens_device, enable_spec_decode ) @@ -862,30 +869,22 @@ def _prepare_fast( buffers["position_ids"][:total_num_tokens], non_blocking=True ) - sequence_lengths = buffers["sequence_lengths"][:num_sequences] - attn_metadata._seq_lens = sequence_lengths - if attn_metadata.is_cuda_graph and attn_metadata._seq_lens_cuda is not None: - attn_metadata._seq_lens_cuda.copy_(sequence_lengths, non_blocking=True) - else: - attn_metadata._seq_lens_cuda = sequence_lengths.cuda(non_blocking=True) - - attn_metadata._num_contexts = scheduled_requests.num_context_requests - attn_metadata._num_ctx_tokens = num_context_tokens - attn_metadata._num_generations = num_generation_requests - attn_metadata._num_tokens = total_num_tokens - attn_metadata.beam_width = 1 - attn_metadata.request_ids = request_ids - attn_metadata.prompt_lens = buffers["prompt_lengths"][:num_sequences] - attn_metadata.num_chunked_ctx_requests = 0 - attn_metadata.kv_cache_params = KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=buffers["cached_token_lengths"][:num_sequences], - num_extra_kv_tokens=0, - ) - attn_metadata.kv_cache_manager = kv_cache_manager assert isinstance(attn_metadata, TrtllmAttentionMetadata) attn_metadata.prepare_encoder_decoder_from_precomputed_lengths( - prompt_lens=buffers["prompt_lengths"][:num_sequences], + BatchInputs( + seq_lens=buffers["sequence_lengths"][:num_sequences], + num_contexts=scheduled_requests.num_context_requests, + request_ids=request_ids, + prompt_lens=buffers["prompt_lengths"][:num_sequences], + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=buffers["cached_token_lengths"][:num_sequences], + num_extra_kv_tokens=0, + ), + ), + num_ctx_tokens=num_context_tokens, + num_generations=num_generation_requests, + num_tokens=total_num_tokens, kv_lens=buffers["kv_lengths"][:num_sequences], context_kv_tokens=context_kv_tokens, generation_kv_tokens=generation_kv_tokens, @@ -1318,7 +1317,6 @@ class EncoderDecoderRunner(DecoderRunner): _context_graph_promotion_supported = False _steady_gen_cache_supported = False _eager_workspace_reclaim_supported = False - _deferred_decode_outputs_supported = False def __init__( self, diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index f12801b255e1..133acb63993f 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -9205,6 +9205,14 @@ def update_resources( self._allocated_draft_lens.pop(req.py_request_id, None) self.try_commit_generation_blocks(req) + def prepare_batch_tables(self, request_ids: List[int], num_contexts: int) -> None: + """Build the per-batch tables ``copy_batch_block_offsets`` reads. + + The attention metadata calls this once per step before it copies the + batch's block offsets. Nothing by default; managers whose offsets are + derived per batch (sliding-window tables) override it. + """ + def copy_batch_block_offsets( self, dst_tensor: torch.Tensor, diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index a21bc9eaec98..a8d9d9243106 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -2747,6 +2747,15 @@ def batch_block_table_key(self, request_ids: List[int]) -> Optional[tuple]: ``KVCacheManagerV2.batch_block_table_key``); None: always copy.""" return None + def prepare_batch_tables(self, request_ids: List[int], + num_contexts: int) -> None: + """Build the per-batch tables ``copy_batch_block_offsets`` reads. + + The attention metadata calls this once per step before it copies the + batch's block offsets. Nothing by default; managers whose offsets are + derived per batch (sliding-window tables) override it. + """ + def copy_batch_block_offsets(self, dst_tensor: torch.Tensor, request_ids: List[int], diff --git a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py index 613a8d8f9353..a02a80715fb0 100644 --- a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py +++ b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py @@ -15,7 +15,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol +from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, Tuple if TYPE_CHECKING: import torch @@ -205,16 +205,11 @@ class RuntimeHooks: # With skip_unchanged_h2d: a changed packed 2-D upload stages only each row's changed column span (a page table that # gained a page re-sends a few bytes, not every padded row), so the pack keeps fitting the in-graph window. h2d_changed_span: bool = False - # A speculative step under the overlap scheduler re-derives its KV-length metadata (on_update_kv_lens) once, - # after the forward's KV-length correction, instead of also before it: the first pass's device writes are all - # rewritten by the second. - kv_lens_update_once: bool = False - # The overlap scheduler's speculative step correction (position_ids += position offsets, kv_lens += KV-length - # offsets) as the model's one launch, called as ``op(positions, position_offsets, kv_lens, kv_lens_offsets)`` on the - # generation rows; it returns False to decline (the two in-tree adds run). - spec_offsets_op: Optional[ - Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], bool] - ] = None + # Attention-metadata extensions the model's metadata carries, attached at creation (the engine's own choice of + # extensions is kept; a subclass of one replaces it), and their creation arguments, each routed by name to the + # extension that takes it, as (name, value) pairs. + metadata_extensions: Tuple[type, ...] = () + metadata_config: Tuple[Tuple[str, Any], ...] = () # Run the overlap executor loop with the execution stream current, so the # loop's own stream work and the forward are ordered on one stream. loop_on_execution_stream: bool = False diff --git a/tensorrt_llm/_torch/pyexecutor/steady_spec_prepare.py b/tensorrt_llm/_torch/pyexecutor/steady_spec_prepare.py index e54aaf57bfca..1b1246dbceb2 100644 --- a/tensorrt_llm/_torch/pyexecutor/steady_spec_prepare.py +++ b/tensorrt_llm/_torch/pyexecutor/steady_spec_prepare.py @@ -26,7 +26,7 @@ from tensorrt_llm._utils import prefer_pinned -from ..attention.backends.interface import AttentionMetadata +from ..attention.backends.interface import AttentionMetadata, BatchInputs from ..attention.backends.trtllm import TrtllmAttentionMetadata from ..speculative.spec_sampler_base import SampleStateTensorsSpec @@ -503,7 +503,13 @@ def stage( positions=buffers.position_ids_cuda, mrope_positions=buffers.mrope_position_ids_cuda if self.use_mrope else None, gather_ids=buffers.gather_ids_cuda, - kv_lens_cuda=attn_metadata.kv_lens_cuda if rec.attn_steady else None, + # The kernel writes the scheduled KV lengths; the steady step that + # follows (prepare_steady_decode_step) takes them as derived. + kv_lens_cuda=( + attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"] + if rec.attn_steady + else None + ), static_input_ids=rec.static_ids, static_position_ids=rec.static_pos, block=self.block, @@ -515,9 +521,11 @@ def stage( event.record() return num_cached, kv_lens, n if rec.static_ids is not None else 0 - def prepare_attention(self, attn_metadata: AttentionMetadata, kv_lens: list[int]) -> None: - """The attention metadata's prepare() of a staged steady step.""" + def prepare_attention( + self, attn_metadata: AttentionMetadata, inputs: BatchInputs, kv_lens: list[int] + ) -> None: + """The attention metadata's prepare of a staged steady step from the step's ``inputs``.""" if self.record.attn_steady: - attn_metadata.prepare_steady_decode_step(kv_lens) + attn_metadata.prepare_steady_decode_step(inputs, kv_lens) else: - attn_metadata.prepare() + attn_metadata.prepare(inputs) diff --git a/tensorrt_llm/_torch/speculative/__init__.py b/tensorrt_llm/_torch/speculative/__init__.py index c3ffe7e28bda..9ede8dce7d41 100644 --- a/tensorrt_llm/_torch/speculative/__init__.py +++ b/tensorrt_llm/_torch/speculative/__init__.py @@ -4,8 +4,6 @@ DraftTargetOneModelWorker) from .eagle3 import MTPEagleWorker from .interface import (SpecMetadata, SpecWorkerBase, draft_prompt_lookahead, - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, should_use_separate_draft_kv_cache) from .mtp import MTPSpecMetadata, MTPWorker from .ngram import NGramDrafter, NGramPoolManager @@ -59,8 +57,6 @@ "get_spec_metadata", "get_spec_resource_manager", "get_spec_worker", - "prepare_attn_metadata_for_draft_replay", - "restore_attn_metadata_after_draft_replay", "should_use_separate_draft_kv_cache", "update_spec_config_from_draft_model_config", "update_spec_config_from_loaded_model", diff --git a/tensorrt_llm/_torch/speculative/dflash.py b/tensorrt_llm/_torch/speculative/dflash.py index b2bd10c8fb2b..5b0c098ab1c3 100644 --- a/tensorrt_llm/_torch/speculative/dflash.py +++ b/tensorrt_llm/_torch/speculative/dflash.py @@ -894,12 +894,6 @@ def __init__( self._saved_ctx_len_host = None self._saved_req_ctx_pos = None self._ctx_len_restore_pending = False - # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, - # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). - self._kv_rewind_pending = False - self._kv_rewind_amount = None - self._kv_rewind_nc = None - self._kv_rewind_bs = None self._batch_to_slot_buffer = None self._max_ctx = 0 self._ctx_k_buf = None # [max_batch+1, L, max_ctx+block, nkv, hd] @@ -1393,17 +1387,22 @@ def _refresh_ctx_block_tables(self, attn_metadata, num_seqs: int) -> bool: """ if self._ctx_block_tables is None or num_seqs <= 0: return False - offsets_name = ( - "kv_cache_block_offsets" if self._ctx_unified_cache else "draft_kv_cache_block_offsets" - ) - src = getattr(attn_metadata, offsets_name, None) + if self._ctx_unified_cache: + # A unified draft KV cache is the target's manager: its block + # offsets are the target's. + src = getattr(attn_metadata, "kv_cache_block_offsets", None) + else: + # The draft metadata over the draft KV cache stages its own block + # offsets, the whole table, in the target's prepare. + draft = attn_metadata.find("draft_or_none", None) + src = draft.kv_cache_block_offsets if draft is not None else None if src is None: # The table is bound but unfillable. Leaving it at zeros would send # every request's every layer to pool block 0 -- silently. raise RuntimeError( "DFlash: bound to the draft KV pool but " f"{type(attn_metadata).__name__} carries no " - f"{offsets_name} this iteration." + "draft KV block offsets this iteration." ) # Field 0 of the K/V pair: both name the same block for a kv_factor=2 # pool, and the drafter addresses blocks, not planes. The block count @@ -1427,13 +1426,18 @@ def _lazy_init_ctx_buffers( draft_kv_cache_manager is not None and draft_kv_cache_manager is getattr(attn_metadata, "kv_cache_manager", None) ) - offsets_name = ( - "kv_cache_block_offsets" if self._ctx_unified_cache else "draft_kv_cache_block_offsets" - ) - if draft_kv_cache_manager is not None and not hasattr(attn_metadata, offsets_name): + # Only TrtllmAttentionMetadata has KV block offsets (and its draft + # metadata's). Without them the table stays all-zero and every request + # would read and write pool block 0 -- silently, at acceptance 1.0. + if self._ctx_unified_cache: + has_block_offsets = hasattr(attn_metadata, "kv_cache_block_offsets") + else: + draft = attn_metadata.find("draft_or_none", None) + has_block_offsets = draft is not None and hasattr(draft, "kv_cache_block_offsets") + if draft_kv_cache_manager is not None and not has_block_offsets: if self._ctx_unified_cache: raise NotImplementedError( - f"DFlash unified KV requires {offsets_name} in attention metadata" + "DFlash unified KV requires kv_cache_block_offsets in attention metadata" ) logger.warning( f"DFlash: {type(attn_metadata).__name__} carries no draft KV block offsets; " @@ -1924,54 +1928,6 @@ def _store_context_kv_paged( kv_layout="HND", ) - def _prepare_attn_metadata_for_dflash(self, attn_metadata, spec_metadata): - """Save attn_metadata fields that DFlash modifies during forward.""" - is_capturing = torch.cuda.is_current_stream_capturing() - - if spec_metadata.is_cuda_graph and not is_capturing: - # Only TRTLLM attention metadata keeps device-side KV lengths; on - # other backends (FlashInfer) there is nothing to save. - fields = ["_seq_lens", "_seq_lens_cuda"] - if hasattr(attn_metadata, "kv_lens_cuda"): - fields.append("kv_lens_cuda") - attn_metadata.prepare_for_spec_dec(*fields) - else: - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - - def _prepare_kv_for_draft_forward( - self, - attn_metadata, - num_accepted_tokens: torch.Tensor, - num_contexts: int, - batch_size: int, - ): - """Adjust kv_lens_cuda so the draft model sees correct RoPE positions.""" - if hasattr(attn_metadata, "kv_lens_cuda"): - self._kv_rewind_amount = 1 - num_accepted_tokens[num_contexts:batch_size] - self._kv_rewind_nc = num_contexts - self._kv_rewind_bs = batch_size - - if batch_size > num_contexts: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 - self._kv_rewind_pending = True - - attn_metadata.update_for_spec_dec() - - def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): - """Apply the deferred kv_lens rewind after the draft forward.""" - self._kv_rewind_pending = False - is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() - if is_warmup: - # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and - # is restored wholesale, so no rewind is needed. - return - - if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): - nc = self._kv_rewind_nc - bs = self._kv_rewind_bs - attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount - attn_metadata.kv_lens_cuda[nc:bs].clamp_(min=0) - def _advance_ctx_len(self, slots: torch.Tensor, num_accepted: torch.Tensor) -> None: """Add this step's accepted tokens to the context lengths of ``slots``, on the device. @@ -2200,16 +2156,7 @@ def _store_prefill_context( self._write_ctx_len(ctx_len_updates, ctx_base_updates) def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - # Restore first (in warmup mode kv_lens_cuda was saved and comes back - # wholesale), then apply any pending rewind for the other modes so a - # failed draft forward does not leave kv_lens_cuda incremented. super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - if ( - getattr(self, "_kv_rewind_pending", False) - and attn_metadata is not None - and spec_metadata is not None - ): - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) if ( getattr(self, "_ctx_len_restore_pending", False) and self._ctx_len is not None @@ -2325,196 +2272,199 @@ def _forward_impl( state_indices=attn_metadata.mamba_metadata.state_indices, ) - self._prepare_attn_metadata_for_dflash(attn_metadata, spec_metadata) - self._prepare_kv_for_draft_forward( - attn_metadata, num_accepted_tokens, num_contexts, batch_size - ) - - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) - - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # The drafter's inputs set their lengths on the draft metadata, which + # leaves the target's untouched; without one, the target's lengths + # are saved and restored around the draft. The drafter attends its + # own context cache and reads no KV lengths. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens - ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, - ) + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] - if num_gens > 0: - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - block_placement=inputs["block_placement"], + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_gather_ids = self._draft_slot_ids(draft_model, num_gens, block_size, K) - # Shields only the last request: at block_size == K with - # shift_label off, slots run 1..K, so every request reads the - # next one's slot 0 and the last overruns. Degrades, never raises. - gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) - - gen_hidden_states = hidden_states_out[gen_gather_ids] - vocab_parallel = self._vocab_parallel - if ( - vocab_parallel is not None - and vocab_parallel.draft_tokens is not None - and not spec_metadata.wants_advanced_draft_sampling - ): - # Greedy draft tokens straight from this rank's vocab shard - # of the block logits, bit-identical to refining the - # gathered logits and taking the per-position argmax. - local_block = vocab_parallel.head.local_logits(gen_hidden_states) - gen_draft_tokens = vocab_parallel.draft_tokens( - draft_model, local_block.view(num_gens, K, -1), inputs - ) - vocab_size = vocab_parallel.head.vocab_size - else: - gen_logits = draft_model.logits_processor( - gen_hidden_states, draft_model.lm_head, attn_metadata, True + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # The logits read the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + block_placement=inputs["block_placement"], ) - # The drafter shares the target's LM head but not its logits - # processor, so apply the transform its config declares, - # before any refinement: a DSpark Markov bias is added to - # the transformed logits. - gen_logits = draft_model.apply_draft_logit_transform(gen_logits) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_gather_ids = self._draft_slot_ids(draft_model, num_gens, block_size, K) + # Shields only the last request: at block_size == K with + # shift_label off, slots run 1..K, so every request reads the + # next one's slot 0 and the last overruns. Degrades, never raises. + gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) + + gen_hidden_states = hidden_states_out[gen_gather_ids] + vocab_parallel = self._vocab_parallel + if ( + vocab_parallel is not None + and vocab_parallel.draft_tokens is not None + and not spec_metadata.wants_advanced_draft_sampling + ): + # Greedy draft tokens straight from this rank's vocab shard + # of the block logits, bit-identical to refining the + # gathered logits and taking the per-position argmax. + local_block = vocab_parallel.head.local_logits(gen_hidden_states) + gen_draft_tokens = vocab_parallel.draft_tokens( + draft_model, local_block.view(num_gens, K, -1), inputs + ) + vocab_size = vocab_parallel.head.vocab_size + else: + gen_logits = draft_model.logits_processor( + gen_hidden_states, draft_model.lm_head, draft_attn_metadata, True + ) + # The drafter shares the target's LM head but not its logits + # processor, so apply the transform its config declares, + # before any refinement: a DSpark Markov bias is added to + # the transformed logits. + gen_logits = draft_model.apply_draft_logit_transform(gen_logits) + + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with - # one coherent path through the block (absent for plain - # DFlash). - if getattr(draft_model, "lilicorr", None) is not None: - gen_logits = self._apply_lilicorr( - draft_model, + # DFlash 2: replace the independent per-position picks with + # one coherent path through the block (absent for plain + # DFlash). + if getattr(draft_model, "lilicorr", None) is not None: + gen_logits = self._apply_lilicorr( + draft_model, + gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["lilicorr_anchor"], + spec_metadata, + ) + vocab_size = gen_logits.shape[-1] + elif getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( + draft_model, + gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + spec_metadata, + ) + vocab_size = gen_logits.shape[-1] + + gen_draft_tokens = self.sample_draft_tokens( gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["lilicorr_anchor"], spec_metadata, + batch_size, + num_contexts=num_contexts, ) - vocab_size = gen_logits.shape[-1] - elif getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( + + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None by default -> no calibration rows collected; + # see accept_stats.py). Guarded off for CUDA-graph batches + # and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], draft_model, - gen_logits, gen_hidden_states.reshape(num_gens, K, -1), inputs["first_prev_tokens"], - spec_metadata, + gen_draft_tokens, ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) - - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None by default -> no calibration rows collected; - # see accept_stats.py). Guarded off for CUDA-graph batches - # and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], - draft_model, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - gen_draft_tokens, - ) - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs( + spec_metadata, num_contexts, num_gens, K, gen_vocab + ) - self._restore_attn_metadata_from_spec_dec(attn_metadata) - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._restore_ctx_len_device() - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._restore_ctx_len_device() + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False - return { - # The gathered float32 target logits, or this rank's vocab shard - # when the target tokens were selected from it. - "logits": raw_logits, - "logits_vocab_shard": logits_vocab_shard, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + return { + # The gathered float32 target logits, or this rank's vocab shard + # when the target tokens were selected from it. + "logits": raw_logits, + "logits_vocab_shard": logits_vocab_shard, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def _draft_block_width(self, draft_model) -> int: """Block slots the draft forward must compute for K draft tokens. @@ -2886,10 +2836,12 @@ def prepare_1st_drafter_inputs( # Update seq_lens for gen requests to K+1 (the number of tokens # the target forward actually processed). - attn_metadata._seq_lens_cuda[num_contexts : num_contexts + num_gens] = ( - total_tokens_per_req + attn_metadata.set_seq_lens( + slice(num_contexts, num_contexts + num_gens), + host=total_tokens_per_req, + device=total_tokens_per_req, ) - attn_metadata._seq_lens[num_contexts : num_contexts + num_gens] = total_tokens_per_req + self.refresh_draft() else: noise_embedding = hidden_states.new_empty(0, 0, hidden_dim) query_positions = torch.empty(0, 0, dtype=torch.long, device="cuda") @@ -2975,7 +2927,8 @@ def _prepare_paged_latent_gen_inputs( block_size, width, ) - attn_metadata._seq_lens[num_contexts : num_contexts + num_gens] = width + attn_metadata.set_seq_lens(slice(num_contexts, num_contexts + num_gens), host=width) + self.refresh_draft() noise_embedding = dflash_noise_block_embedding( draft_model.draft_model_full.model.embed_tokens, diff --git a/tensorrt_llm/_torch/speculative/draft_target.py b/tensorrt_llm/_torch/speculative/draft_target.py index 1b853b0bf1fb..8202443a9d4d 100644 --- a/tensorrt_llm/_torch/speculative/draft_target.py +++ b/tensorrt_llm/_torch/speculative/draft_target.py @@ -26,6 +26,7 @@ import torch from torch import nn +from tensorrt_llm._torch.attention.backends.features.spec_decoding import set_use_spec_decoding from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.mapping import Mapping @@ -91,30 +92,6 @@ def __init__( def max_draft_len(self) -> int: return self.spec_config.max_draft_len - def _prepare_attn_metadata_for_draft_target( - self, - attn_metadata: AttentionMetadata, - spec_metadata: DraftTargetOneModelSpecMetadata, - ): - """ - Save the attention metadata fields modified by DraftTarget. - - During CUDA-graph warmup, kv_lens_cuda is also saved/restored to avoid - cross-warmup accumulation. During capture and normal inference we keep - kv_lens_cuda live so the updates persist. - """ - is_capturing = torch.cuda.is_current_stream_capturing() - - if ( - spec_metadata.is_cuda_graph - and not is_capturing - and hasattr(attn_metadata, "kv_lens_cuda") - and isinstance(attn_metadata.kv_lens_cuda, torch.Tensor) - ): - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda", "kv_lens_cuda") - else: - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - def _update_kv_after_first_draft_step( self, attn_metadata: AttentionMetadata, @@ -123,15 +100,15 @@ def _update_kv_after_first_draft_step( batch_size: int, runtime_draft_len: int, ): - if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - runtime_draft_len - num_accepted_tokens[num_contexts:batch_size] - ) - attn_metadata.kv_lens_cuda[:num_contexts] += 1 - # Some attention backends keep extra indexing state derived from # seq_lens / kv_lens that must be refreshed for chained drafting. - attn_metadata.update_for_spec_dec() + if hasattr(attn_metadata, "kv_lens_cuda"): + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, + ) + attn_metadata.adjust_kv_lens(slice(0, num_contexts), 1) + self.refresh_draft() def _update_kv_for_chained_draft_step( self, @@ -139,9 +116,8 @@ def _update_kv_for_chained_draft_step( batch_size: int, ): if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[:batch_size] += 1 - - attn_metadata.update_for_spec_dec() + attn_metadata.adjust_kv_lens(slice(0, batch_size), 1) + self.refresh_draft() def _forward_impl( self, @@ -187,126 +163,131 @@ def _forward_impl( logits, attn_metadata, spec_metadata ) - # Prepare attention metadata for speculative decoding and save state for restore - self._prepare_attn_metadata_for_draft_target(attn_metadata, spec_metadata) - - # Prepare inputs for the first draft forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - ) + # The draft forwards run on the draft metadata, which leaves the + # target's untouched; without one, the target's state is saved and + # restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the first draft forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + ) - next_draft_tokens = [] - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - for i in range(runtime_draft_len): - if i == 0: - start_ids_gen = ( - spec_metadata.batch_indices_cuda[:num_gens] * (runtime_draft_len + 1) - ).long() - gather_ids_gen = ( - start_ids_gen - + num_accepted_tokens[num_contexts:] - - 1 - + attn_metadata.num_ctx_tokens - ) - gather_ids = torch.concat( - [spec_metadata.gather_ids[:num_contexts], gather_ids_gen], dim=0 - ) - else: - gather_ids = spec_metadata.batch_indices_cuda[:batch_size] + next_draft_tokens = [] + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - if self.guided_decoder is not None: - new_tokens = inputs["input_ids"][gather_ids] - self.guided_decoder.add_draft_batch( - new_tokens, num_accepted_tokens, draft_step=i - ) + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - if original_all_rank_num_tokens is not None: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + inputs["attn_metadata"] = draft_attn_metadata + for i in range(runtime_draft_len): if i == 0: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens - elif spec_metadata.all_rank_num_seqs is not None: - attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_seqs - - hidden_states = draft_model.model(**inputs) - if isinstance(hidden_states, tuple): - hidden_states = hidden_states[0] - - # Disable spec-dec mode for chained draft steps - attn_metadata.use_spec_decoding = False - - logits = draft_model.logits_processor( - hidden_states[gather_ids], draft_model.lm_head, attn_metadata, True - ) - if self.guided_decoder is not None: - self.guided_decoder.execute_draft_batch(logits, self._d2t, draft_step=i) - - new_draft_token = self.sample_draft_tokens( - logits, - spec_metadata, - batch_size, - draft_step=i, - ) - next_draft_tokens.append(new_draft_token) - - # Update inputs and metadata for next draft step - position_ids = inputs["position_ids"][gather_ids] + 1 - if i == 0: - attn_metadata._seq_lens[:batch_size].fill_(1) - attn_metadata._seq_lens_cuda[:batch_size].fill_(1) - attn_metadata.on_update() - if inputs["attn_metadata"].kv_cache_manager is not None: - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) - attn_metadata.num_contexts = 0 - self._update_kv_after_first_draft_step( - attn_metadata, - num_accepted_tokens, - num_contexts, + start_ids_gen = ( + spec_metadata.batch_indices_cuda[:num_gens] * (runtime_draft_len + 1) + ).long() + gather_ids_gen = ( + start_ids_gen + + num_accepted_tokens[num_contexts:] + - 1 + + draft_attn_metadata.num_ctx_tokens + ) + gather_ids = torch.concat( + [spec_metadata.gather_ids[:num_contexts], gather_ids_gen], dim=0 + ) + else: + gather_ids = spec_metadata.batch_indices_cuda[:batch_size] + + if self.guided_decoder is not None: + new_tokens = inputs["input_ids"][gather_ids] + self.guided_decoder.add_draft_batch( + new_tokens, num_accepted_tokens, draft_step=i + ) + + if original_all_rank_num_tokens is not None: + if i == 0: + draft_attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + elif spec_metadata.all_rank_num_seqs is not None: + draft_attn_metadata.all_rank_num_tokens = ( + spec_metadata.all_rank_num_seqs + ) + + hidden_states = draft_model.model(**inputs) + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + + # Disable spec-dec mode for chained draft steps + set_use_spec_decoding(draft_attn_metadata, False) + + logits = draft_model.logits_processor( + hidden_states[gather_ids], draft_model.lm_head, draft_attn_metadata, True + ) + if self.guided_decoder is not None: + self.guided_decoder.execute_draft_batch(logits, self._d2t, draft_step=i) + + new_draft_token = self.sample_draft_tokens( + logits, + spec_metadata, batch_size, - runtime_draft_len, + draft_step=i, ) - else: - self._update_kv_for_chained_draft_step(attn_metadata, batch_size) - - inputs = { - "input_ids": new_draft_token, - "position_ids": position_ids, - "attn_metadata": attn_metadata, - "spec_metadata": spec_metadata, - } - - next_draft_tokens = torch.stack(next_draft_tokens, dim=1) - - # Restore attention metadata to original state - self._restore_attn_metadata_from_spec_dec(attn_metadata) - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens - - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_draft_tokens.append(new_draft_token) - attn_metadata.use_spec_decoding = True + # Update inputs and metadata for next draft step + position_ids = inputs["position_ids"][gather_ids] + 1 + if i == 0: + draft_attn_metadata.set_seq_lens( + slice(0, batch_size), + host=1, + device=1, + update_counts=True, + ) + if inputs["attn_metadata"].kv_cache_manager is not None: + # The context requests continue as one-token + # generation rows. + draft_attn_metadata.as_generation_rows() + self._update_kv_after_first_draft_step( + draft_attn_metadata, + num_accepted_tokens, + num_contexts, + batch_size, + runtime_draft_len, + ) + else: + self._update_kv_for_chained_draft_step(draft_attn_metadata, batch_size) + + inputs = { + "input_ids": new_draft_token, + "position_ids": position_ids, + "attn_metadata": draft_attn_metadata, + "spec_metadata": spec_metadata, + } + + next_draft_tokens = torch.stack(next_draft_tokens, dim=1) + + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + set_use_spec_decoding(attn_metadata, True) + + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def prepare_1st_drafter_inputs( self, diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index 744a82a1d361..940b336ce611 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -10,13 +10,16 @@ from torch import nn from tensorrt_llm._device_copy import DeviceCopyBuffer +from tensorrt_llm._torch.attention.backends.features.spec_decoding import \ + set_use_spec_decoding from tensorrt_llm._torch.custom_ops import inplace_slice_copy from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.mapping import Mapping from ..attention.backends import AttentionMetadata +from ..attention.backends.features import FlashMlaMetadata from ..attention.backends.flashinfer import FlashInferAttentionMetadata -from ..attention.backends.sparse.params import MTPIndexShareMetadata +from ..attention.backends.sparse.params import MTPIndexShareExtension from ..model_config import ModelConfig from ..pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManager from ..pyexecutor.llm_request import LlmRequest @@ -33,13 +36,6 @@ from ...llmapi.llm_args import EagleDecodingConfig -def _reset_mtp_index_share(attn_metadata: MTPIndexShareMetadata) -> None: - """Clear the draft-loop state a sparse indexer reads.""" - attn_metadata.set_skip_topk(False) - attn_metadata.set_in_mtp_draft_loop(False) - attn_metadata.set_mtp_num_accepted(None) - - class Eagle3ResourceManager(BaseResourceManager): """ Eagle3 needs to save the hidden states for the draft model. Use this class @@ -488,16 +484,6 @@ def __init__(self, # MTP Eagle: lazily-resolved flag for Mamba hybrid cache support self._is_mamba_hybrid_cache = None - # Worker-side saved spec-dec params; initialized so that a failure - # inside _prepare_attn_metadata_for_spec_dec (e.g. an OOM in the - # clone calls) cannot turn the cleanup restore into AttributeError. - self._saved_packed_mask = None - self._saved_position_offsets = None - self._saved_spec_decoding_query_len = None - self._saved_generation_lengths = None - # Set by set_draft_model: the MTP layer applies the draft loop's - # deferred kv_lens increment itself. - self._draft_folds_kv_lens_increment = False # MXFP8 copy of the LM head for greedy draft tokens (MTP Eagle), set # by the model once its weights are loaded. self.draft_head = None @@ -505,14 +491,6 @@ def __init__(self, and supports_sharded_target_sampling( spec_config, mapping)) - def set_draft_model(self, draft_model) -> None: - super().set_draft_model(draft_model) - # Import here to avoid a circular import. - from ..models.modeling_speculative import MTPForCausalLM - self._draft_folds_kv_lens_increment = ( - self.is_mtp_eagle and isinstance(draft_model, MTPForCausalLM) - and draft_model.folds_kv_lens_increment) - @property def max_draft_len(self) -> int: return self.spec_config.max_draft_len @@ -538,54 +516,6 @@ def target_logits(self, hidden_states: torch.Tensor, lm_head, return super().target_logits(hidden_states, lm_head, logits_processor, attn_metadata, spec_metadata, draft_model) - def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - batch_size = attn_metadata.num_seqs - - # Save spec-dec params that the drafting loop will overwrite. - # Without this, CUDA graph warmup's second iteration would run - # the target model attention with stale draft-layer masks - # instead of the correct target-tree masks. - if attn_metadata.spec_decoding_packed_mask is not None: - self._saved_packed_mask = attn_metadata.spec_decoding_packed_mask[: - batch_size].clone( - ) - else: - self._saved_packed_mask = None - if attn_metadata.spec_decoding_position_offsets is not None: - self._saved_position_offsets = attn_metadata.spec_decoding_position_offsets.clone( - ) - self._saved_spec_decoding_query_len = ( - attn_metadata.spec_decoding_query_len) - else: - self._saved_position_offsets = None - self._saved_spec_decoding_query_len = None - if attn_metadata.spec_decoding_generation_lengths is not None: - self._saved_generation_lengths = attn_metadata.spec_decoding_generation_lengths[:batch_size].clone( - ) - else: - self._saved_generation_lengths = None - - def _restore_attn_metadata_from_spec_dec(self, attn_metadata): - super()._restore_attn_metadata_from_spec_dec(attn_metadata) - if self._saved_packed_mask is not None: - batch_size = self._saved_packed_mask.shape[0] - attn_metadata.spec_decoding_packed_mask[:batch_size].copy_( - self._saved_packed_mask) - self._saved_packed_mask = None - if self._saved_position_offsets is not None: - attn_metadata.spec_decoding_position_offsets.copy_( - self._saved_position_offsets) - attn_metadata.spec_decoding_query_len = ( - self._saved_spec_decoding_query_len) - self._saved_position_offsets = None - self._saved_spec_decoding_query_len = None - if self._saved_generation_lengths is not None: - batch_size = self._saved_generation_lengths.shape[0] - attn_metadata.spec_decoding_generation_lengths[:batch_size].copy_( - self._saved_generation_lengths) - self._saved_generation_lengths = None - # Skip torch.compile for now since current Torch is not compatible with Triton 3.4 # @torch.compile(options={"max-autotune": True}) @@ -665,41 +595,39 @@ def _forward_impl(self, batch_size=batch_size, )) else: - # The step-0 sequence lengths, before the draft loop rewrites them. - pre_draft_seq_lens = attn_metadata._seq_lens_cuda - # Save the old attn_metadata and spec_metadata - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - - # Prepare inputs for the 1st draft model forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model) - - # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here - # so the post-loop restore (below) can put attn_metadata back into a - # state the target model expects. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager( - resource_manager) - - next_draft_tokens, next_new_tokens = self._forward_draft_loop( - inputs, attn_metadata, spec_metadata, draft_model, - draft_kv_cache_manager, num_contexts, num_gens, batch_size, - num_accepted_tokens, original_all_rank_num_tokens, - resource_manager, accepted_tokens, pre_draft_seq_lens) - # restore attn_metadata to support cuda graph - self._restore_attn_metadata_from_spec_dec(attn_metadata) - # restore all_rank_num_tokens for attention DP - if original_all_rank_num_tokens is not None: - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + # The step-0 sequence lengths, before the draft loop changes them: + # the target's, which the draft loop never changes when it runs on + # the draft metadata. + pre_draft_seq_lens = attn_metadata.seq_lens_cuda + # The draft forwards run on the draft metadata, which leaves the + # target's untouched; without one, the target's state is saved + # and restored around the draft loop. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + # Prepare inputs for the 1st draft model forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model) + + # Predict draft tokens. ``original_all_rank_num_tokens`` is saved here + # so the post-loop restore (below) can put attn_metadata back into a + # state the target model expects. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager( + resource_manager) + + next_draft_tokens, next_new_tokens = self._forward_draft_loop( + inputs, draft_attn_metadata, spec_metadata, draft_model, + draft_kv_cache_manager, num_contexts, num_gens, batch_size, + num_accepted_tokens, original_all_rank_num_tokens, + resource_manager, accepted_tokens, pre_draft_seq_lens) # prepare next new tokens to support overlap scheduler; the draft loop # may have built them already as a view of its token buffer. @@ -709,7 +637,7 @@ def _forward_impl(self, spec_metadata.batch_indices_cuda, batch_size, num_accepted_tokens) - attn_metadata.use_spec_decoding = True + set_use_spec_decoding(attn_metadata, True) # Commit model-owned target state only after draft preparation has # succeeded; SpecWorkerBase aborts the still-pending snapshots on any @@ -771,7 +699,7 @@ def _forward_external_shared_target_kv_draft_loop( draft_metadata.update_shared_kv_draft_lengths(attn_metadata, num_accepted_tokens, num_contexts) - draft_metadata.use_spec_decoding = False + set_use_spec_decoding(draft_metadata, False) draft_metadata.padded_num_tokens = None draft_metadata.all_rank_num_tokens = ( spec_metadata.subseq_all_rank_num_tokens) @@ -843,8 +771,6 @@ def _forward_linear_draft_loop(self, accepted_tokens=None, pre_draft_seq_lens=None): """Linear draft loop, unified for Eagle3 and MTP Eagle.""" - from ..attention.backends.sparse.dsa import is_dsa_cache_manager - runtime_draft_len = spec_metadata.runtime_draft_len num_gens = batch_size - num_contexts next_draft_tokens = [] @@ -876,36 +802,28 @@ def _forward_linear_draft_loop(self, # Sparse backends that can reuse one indexer selection across the draft # loop expose this setter; each indexer still decides whether its own # config turns the reuse on. - uses_mtp_index_share = self.is_mtp_eagle and isinstance( - attn_metadata, MTPIndexShareMetadata) + index_share = (attn_metadata.find_ext(MTPIndexShareExtension) + if self.is_mtp_eagle else None) with contextlib.ExitStack() as draft_scope: - if uses_mtp_index_share: - attn_metadata.set_in_mtp_draft_loop(True) + if index_share is not None: + index_share.set_in_mtp_draft_loop(True) # Accepted counts let the indexer stash each gen's last-accepted row. - attn_metadata.set_mtp_num_accepted(num_accepted_tokens) + index_share.set_mtp_num_accepted(num_accepted_tokens) # The metadata outlives this call, so clear the state even when a # draft step raises: a stale draft-loop flag would make the next # target forward reuse these selections. The loop writes - # set_skip_topk on the metadata yielded below, which is this same - # object: draft_kv_cache_context swaps buffers in place for every - # TrtllmAttentionMetadata, and sparse metadata all derive from it. - draft_scope.callback(_reset_mtp_index_share, attn_metadata) + # set_skip_topk on the extension of the metadata yielded below, + # which is this same object: draft_kv_cache_context yields a + # draft metadata as it is. + draft_scope.callback(index_share.reset_mtp_index_share) draft_attn_metadata = draft_scope.enter_context( self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager)) attn_metadata = draft_attn_metadata inputs["attn_metadata"] = draft_attn_metadata - if self.is_mtp_eagle and is_dsa_cache_manager( - draft_kv_cache_manager): - # Overlap scheduling corrects kv_lens_cuda from the runtime - # accepted-token counts inside the captured graph. The target - # forward refreshes target slot mappings during that correction; - # refresh once more after rebinding so the first draft forward - # writes the separate DSA indexer cache at the same positions. - attn_metadata.on_update_kv_lens() for i in range(runtime_draft_len): - if uses_mtp_index_share: - attn_metadata.set_skip_topk(i > 0) + if index_share is not None: + index_share.set_skip_topk(i > 0) # Run draft model (mode-specific via helper). The helper # passes ``all_rank_num_tokens`` as a kwarg so the draft model # handles save/restore internally (Eagle3DraftModel.forward @@ -922,12 +840,15 @@ def _forward_linear_draft_loop(self, # ``[gather_ids]`` site below takes a slice as readily as an # index tensor. if i == 0 and use_epilogue: + # The epilogue writes the draft's device lengths: the + # buffers come from the core, which marks them changed. + lengths = attn_metadata.lengths_for_kernel_update() step0 = step0_epilogue(pre_draft_seq_lens, num_accepted_tokens, hidden_states, inputs["position_ids"], - attn_metadata._seq_lens_cuda, - attn_metadata.kv_lens_cuda, + lengths["seq_lens_cuda"], + lengths["kv_lens_cuda"], num_contexts, batch_size, runtime_draft_len, @@ -1131,46 +1052,43 @@ def _forward_linear_draft_loop(self, # Update attn_metadata for the next iteration. if i == 0: - attn_metadata._seq_lens[:batch_size].fill_(1) - if step0 is None: # else: done by the step-0 epilogue - attn_metadata._seq_lens_cuda[:batch_size].fill_(1) - attn_metadata.on_update() + # The step-0 epilogue already set the device lengths. + attn_metadata.set_seq_lens( + slice(0, batch_size), + host=1, + device=1 if step0 is None else None, + update_counts=True) has_kv_cache = inputs[ "attn_metadata"].kv_cache_manager is not None - if has_kv_cache and hasattr(attn_metadata, - "host_request_types"): - attn_metadata.host_request_types[:attn_metadata. - num_contexts].fill_(1) if has_kv_cache: - attn_metadata.num_contexts = 0 - if hasattr(attn_metadata, 'kv_lens_cuda') and step0 is None: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - runtime_draft_len - - num_accepted_tokens[num_contexts:]) - attn_metadata.kv_lens_cuda[:num_contexts] += 1 - + # The context requests continue as one-token + # generation rows. + attn_metadata.as_generation_rows() if has_kv_cache: self._prepare_flash_mla_generation_layout( attn_metadata, num_contexts, batch_size) if hasattr(attn_metadata, 'kv_lens_cuda'): - attn_metadata.update_for_spec_dec() + # The step-0 epilogue already corrected the device KV + # lengths. + if step0 is None: + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - + runtime_draft_len) + attn_metadata.adjust_kv_lens( + slice(0, num_contexts), 1) + self.refresh_draft() # Both Eagle3 and MTP Eagle drafters take ``draft_len + 1`` # tokens in the first draft step (attention runs in spec-dec # mode), then 1 token per step in subsequent iterations. # Disable spec_decoding here so the masks/positions stay # correct on subsequent iters. - attn_metadata.use_spec_decoding = False + set_use_spec_decoding(attn_metadata, False) else: if hasattr(attn_metadata, 'kv_lens_cuda'): - if (step0 is not None and i + 1 < runtime_draft_len - and self._draft_folds_kv_lens_increment): - # The next MTP pass applies it (in its fused - # input launch when it takes one). - inputs["kv_lens_increment"] = batch_size - else: - attn_metadata.kv_lens_cuda[:batch_size] += 1 - attn_metadata.update_for_spec_dec() + attn_metadata.adjust_kv_lens(slice(0, batch_size), 1) + self.refresh_draft() next_new_tokens = None if token_buffer is not None: next_draft_tokens = token_buffer[1:].t() @@ -1238,7 +1156,8 @@ def _prepare_flash_mla_generation_layout(self, attn_metadata, num_contexts, With no context rows ``prepare_flash_mla`` already staged this exact layout, so the early return merely skips a redundant copy. """ - if num_contexts <= 0 or not attn_metadata.enable_flash_mla: + if (num_contexts <= 0 + or attn_metadata.find_ext(FlashMlaMetadata) is None): return attn_metadata.block_ids_per_seq[:batch_size].copy_( attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True) diff --git a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py index a5930e0a90dc..a86a6b59a8c8 100644 --- a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py +++ b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py @@ -571,9 +571,12 @@ def prepare_1st_drafter_inputs( dim=0, ) - attn_metadata._seq_lens[num_contexts:batch_size].fill_(max_path_len) - attn_metadata._seq_lens_cuda[num_contexts:batch_size].fill_(max_path_len) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(num_contexts, batch_size), + host=max_path_len, + device=max_path_len, + update_counts=True, + ) else: # Context-only (warmup). No gen tokens. input_ids = input_ids_ctx @@ -643,15 +646,21 @@ def _forward_draft_loop( # KV correction: target processed tokens_per_gen_step per request, # but step 0 should only attend to max_draft_len + 1 accepted-path tokens. if num_gens > 0 and hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= self._kv_correction - - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): + attn_metadata.adjust_kv_lens(slice(num_contexts, batch_size), -self._kv_correction) + self.refresh_draft() + + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata hidden_states, hidden_states_to_save = draft_model.model(**inputs) step0_hs = hidden_states_to_save[gather_ids] logits = draft_model.logits_processor( - hidden_states[gather_ids], draft_model.lm_head, attn_metadata, True + hidden_states[gather_ids], draft_model.lm_head, draft_attn_metadata, True ) new_draft_tokens, new_draft_scores = self.sample(logits, self.K) @@ -662,7 +671,7 @@ def _forward_draft_loop( new_draft_scores=new_draft_scores, previous_draft_scores=None, batch_size=batch_size, - attn_metadata=attn_metadata, + attn_metadata=draft_attn_metadata, ) self.update_hidden_states( @@ -673,7 +682,7 @@ def _forward_draft_loop( self.prepare_for_generation( 0, - attn_metadata, + draft_attn_metadata, batch_size, inputs=inputs, gather_ids=gather_ids, @@ -687,7 +696,7 @@ def _forward_draft_loop( if original_all_rank_num_tokens is not None: if spec_metadata.all_rank_num_seqs is not None: - attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_seqs + draft_attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_seqs # Growing context: process ALL accumulated tokens num_infer_tokens = batch_size * num_tokens_per_req @@ -701,7 +710,7 @@ def _forward_draft_loop( "input_ids": inp_ids, "position_ids": inp_pos, "hidden_states": inp_hs, - "attn_metadata": attn_metadata, + "attn_metadata": draft_attn_metadata, "spec_metadata": spec_metadata, } @@ -711,7 +720,7 @@ def _forward_draft_loop( hs_reshaped = hidden_states.reshape(batch_size, num_tokens_per_req, -1) selected_hs = hs_reshaped[:, -self.K :, :].reshape(batch_size * self.K, -1) logits = draft_model.logits_processor( - selected_hs, draft_model.lm_head, attn_metadata, True + selected_hs, draft_model.lm_head, draft_attn_metadata, True ) new_draft_tokens, new_draft_scores = self.sample(logits, self.K) @@ -726,7 +735,7 @@ def _forward_draft_loop( new_draft_scores=new_draft_scores, previous_draft_scores=previous_draft_scores, batch_size=batch_size, - attn_metadata=attn_metadata, + attn_metadata=draft_attn_metadata, ) self.update_hidden_states( @@ -736,7 +745,7 @@ def _forward_draft_loop( selected_parents=self._last_selected_parents, ) - self.prepare_for_generation(layer_idx, attn_metadata, batch_size) + self.prepare_for_generation(layer_idx, draft_attn_metadata, batch_size) # Resample final tokens and build tree real_draft_tokens, topk_score_indices = self.resampling_final_draft_tokens(batch_size) @@ -1197,21 +1206,25 @@ def prepare_for_generation( -1, self.K ) - attn_metadata._seq_lens[:batch_size].fill_(self.K) - attn_metadata._seq_lens_cuda[:batch_size].fill_(self.K) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(0, batch_size), + host=self.K, + device=self.K, + update_counts=True, + ) if inputs["attn_metadata"].kv_cache_manager is not None: - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) - attn_metadata.num_contexts = 0 + # The context requests continue as generation rows. + attn_metadata.as_generation_rows() if hasattr(attn_metadata, "kv_lens_cuda"): # KV rewind: remove unaccepted draft-path tokens if num_gens > 0: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - self._max_path_len - ) - num_accepted_tokens[num_contexts:batch_size] - attn_metadata.kv_lens_cuda[:batch_size] += self.K + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - self._max_path_len, + ) + attn_metadata.adjust_kv_lens(slice(0, batch_size), self.K) attn_metadata.use_spec_decoding = True @@ -1222,10 +1235,14 @@ def prepare_for_generation( :batch_size, num_tokens_previous_layer:num_tokens_current_layer ] = prev_pos[:, -self.K :] + 1 - attn_metadata._seq_lens[:batch_size].fill_(num_tokens_current_layer) - attn_metadata._seq_lens_cuda[:batch_size].fill_(num_tokens_current_layer) - attn_metadata.on_update() - attn_metadata.kv_lens_cuda[:batch_size] += self.K + attn_metadata.set_seq_lens( + slice(0, batch_size), + host=num_tokens_current_layer, + device=num_tokens_current_layer, + update_counts=True, + ) + attn_metadata.adjust_kv_lens(slice(0, batch_size), self.K) + self.refresh_draft() def update_hidden_states( self, diff --git a/tensorrt_llm/_torch/speculative/interface.py b/tensorrt_llm/_torch/speculative/interface.py index d68ebfff9b8d..88cba759c223 100644 --- a/tensorrt_llm/_torch/speculative/interface.py +++ b/tensorrt_llm/_torch/speculative/interface.py @@ -15,6 +15,7 @@ import copy import os +import weakref from abc import ABC, abstractmethod from contextlib import contextmanager from dataclasses import dataclass, field @@ -29,11 +30,11 @@ from ..._utils import get_sm_version, prefer_pinned from ..attention.backends.interface import AttentionMetadata -from ..attention.backends.trtllm import (AttentionBackend, TrtllmAttention, - TrtllmAttentionMetadata) +from ..attention.backends.trtllm import AttentionBackend, TrtllmAttention from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE from ..pyexecutor import step_inputs from ..pyexecutor.resource_manager import ResourceManagerType +from ..utils import get_model_extra_attrs if TYPE_CHECKING: from ..pyexecutor.guided_decoder import CapturableGuidedDecoder @@ -225,70 +226,6 @@ def draft_prompt_lookahead(spec_config) -> Optional[int]: return None -def prepare_attn_metadata_for_draft_replay(attn_metadata, - draft_kv_cache_manager): - """ - Prepare attention metadata for a draft forward or CUDA graph replay when using a - separate draft KV cache. Swaps cache-layout-dependent buffers, refreshes FlashMLA - block IDs outside capture, and (for DSA) re-prepares indexer slot mappings - for the current batch. - Call restore_attn_metadata_after_draft_replay in a finally block. - Returns saved state or None if no-op. - """ - if (draft_kv_cache_manager is None - or draft_kv_cache_manager is attn_metadata.kv_cache_manager): - return None - if not isinstance(attn_metadata, TrtllmAttentionMetadata): - return None - draft_block_offsets = getattr(attn_metadata, 'draft_kv_cache_block_offsets', - None) - if draft_block_offsets is None: - return None - - # Marks that a swap happened and carries the backend's state for its - # restore hook; swap_for_draft records the originals on attn_metadata. - saved = {} - if attn_metadata.enable_flash_mla: - if (attn_metadata.draft_block_ids_per_seq is None - or attn_metadata.draft_kv_block_ids_per_seq is None): - raise RuntimeError( - "FlashMLA separate draft KV cache requires dedicated draft block-ID buffers" - ) - attn_metadata.swap_for_draft('block_ids_per_seq', - attn_metadata.draft_block_ids_per_seq) - attn_metadata.swap_for_draft('kv_block_ids_per_seq', - attn_metadata.draft_kv_block_ids_per_seq) - attn_metadata.swap_for_draft('kv_cache_manager', draft_kv_cache_manager) - attn_metadata.swap_for_draft('kv_cache_block_offsets', - attn_metadata.draft_kv_cache_block_offsets) - attn_metadata.swap_for_draft( - 'host_kv_cache_block_offsets', - draft_kv_cache_manager.host_kv_cache_block_offsets) - if attn_metadata.enable_flash_mla: - attn_metadata.prepare_flash_mla() - - # Backends select any additional draft-forward state, such as native DSA - # indexer buffers or DeepSeek-V4 sparse tables and pool pointers. - backend_saved = attn_metadata.prepare_for_draft_forward() - if backend_saved is not None: - saved['saved_backend_state'] = backend_saved - return saved - - -def restore_attn_metadata_after_draft_replay(attn_metadata, saved_state): - """Restore attention metadata after draft replay. No-op if saved_state is None.""" - if saved_state is None: - return - attn_metadata.restore_draft_swaps() - if attn_metadata.enable_flash_mla: - # Target and draft block-ID buffers are independent. Restoring only - # needs to invalidate the scheduler metadata; refreshing the unchanged - # target buffers would repeat request-specific H2D work. - attn_metadata._flash_mla_metadata_valid = False - attn_metadata.restore_after_draft_forward( - saved_state.get('saved_backend_state')) - - def get_force_num_accepted_tokens() -> int: """ Read and parse the TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS environment @@ -1808,6 +1745,9 @@ class SpecWorkerBase(nn.Module, ABC): Provides common functionality for sampling and token handling. """ + # The target metadata of the running draft_loop(), or None. + _draft_loop_target = None + def __init__(self, use_separate_draft_kv_cache: bool = False, key_exchange=None): @@ -1889,9 +1829,7 @@ def forward(self, *args, **kwargs): Tolerated forward failures (e.g. an OOM during the max-shape general warmup, or an error-budget-tolerated serving exception) must not leak - the attn-metadata state saved by prepare_for_spec_dec: a stale save - fails every subsequent forward at the pairing assert. - https://nvbugs/6442074 + a worker's transient state into later forwards. """ attn_metadata = kwargs.get("attn_metadata") spec_metadata = kwargs.get("spec_metadata") @@ -1930,17 +1868,12 @@ def target_logits(self, hidden_states: torch.Tensor, lm_head, True) def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - """Restore attn-metadata spec-dec state if a failure skipped it. + """Undo a worker's transient state if a failed forward skipped it. - No-op on the success path: workers restore at their preferred point - and this sees no saved state. Subclasses with extra transient state - (e.g. the deferred kv_lens rewind in PARD/DFlash) extend this. + The draft loop changes only ``md.draft``, so the target metadata has + nothing to restore; subclasses with transient state of their own + (e.g. DFlash's context lengths) extend this. """ - if attn_metadata is not None and attn_metadata.has_spec_dec_saved_state: - logger.warning( - "Spec-dec worker forward failed between prepare_for_spec_dec " - "and restore_from_spec_dec; restoring attn metadata state.") - self._restore_attn_metadata_from_spec_dec(attn_metadata) @property @abstractmethod @@ -2063,20 +1996,6 @@ def target_vocab_parallel_head(self) -> Optional[VocabParallelHead]: place of the gathered target logits. None to receive gathered logits.""" return None - def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): - """ - Prepare attention metadata before speculative decoding draft token generation. - Saves current state for later restoration. - """ - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - - def _restore_attn_metadata_from_spec_dec(self, attn_metadata): - """ - Restore attention metadata after speculative decoding draft token generation. - """ - attn_metadata.restore_from_spec_dec() - attn_metadata.on_update() - def _ensure_force_accept_rng_state(self, device: torch.device) -> None: """ Lazily build the deterministic RNG state used by @@ -3455,41 +3374,69 @@ def get_draft_kv_cache_manager(self, resource_manager): return None @contextmanager - def draft_kv_cache_context(self, attn_metadata, draft_kv_cache_manager): - """ - Select draft attention metadata for one-engine speculative decoding. - - TRTLLM metadata temporarily swaps its manager and cache-layout-dependent - buffers, including DSA indexer offsets and slot mappings. - FlashInfer uses an independently planned metadata view because its page - tables and kernel wrappers are manager-specific. - """ - - # draft_kv_cache_manager is None if using two-engine speculative decoding or not enabling separate draft KV cache. - if (draft_kv_cache_manager is None - or draft_kv_cache_manager is attn_metadata.kv_cache_manager): - yield attn_metadata - return - - from ..attention.backends.flashinfer import FlashInferAttentionMetadata - if isinstance(attn_metadata, FlashInferAttentionMetadata): - yield attn_metadata.get_draft_metadata(draft_kv_cache_manager) - return - - if not isinstance(attn_metadata, TrtllmAttentionMetadata): + def draft_loop(self, attn_metadata): + """Run one draft loop on the draft metadata of ``attn_metadata``. + + ``with md.draft_loop() as draft`` of the target's ``DraftMetadata``: + the draft starts from the target's state (an adopting draft takes its + prepared state, any other draft its lengths, counts and spec-dec + state), and the loop changes only the draft. While the loop runs, the + draft is the model's ``attention_metadata`` extra attribute, which + attention custom ops (MLA) read instead of taking the metadata as an + argument, and ``refresh_draft()`` refreshes it. A stub metadata (no + ``AttentionMetadata``, in unit tests) is yielded as it is. + """ + if not isinstance(attn_metadata, AttentionMetadata): yield attn_metadata return + if attn_metadata.find("draft_or_none", None) is None: + raise RuntimeError( + "One-model drafting runs on md.draft: create the attention " + "metadata with one_model_draft=True") + previous = self._draft_loop_target + with attn_metadata.draft_loop() as draft, \ + self._as_attention_metadata(draft): + self._draft_loop_target = attn_metadata + try: + yield draft + finally: + self._draft_loop_target = previous + + def refresh_draft(self) -> None: + """Refresh the running draft loop's draft before its next forward. + + ``md.refresh_draft()`` of the loop's target: re-derives the draft's + length-dependent state when the loop changed its lengths. A no-op + outside a loop on a stub metadata. + """ + if self._draft_loop_target is not None: + self._draft_loop_target.refresh_draft() - saved_state = prepare_attn_metadata_for_draft_replay( - attn_metadata, draft_kv_cache_manager) - if saved_state is None: + @contextmanager + def _as_attention_metadata(self, attn_metadata): + """Make ``attn_metadata`` the model's ``attention_metadata`` extra + attribute while the context is open.""" + attrs = get_model_extra_attrs() + current = None if attrs is None else attrs.get("attention_metadata") + if current is None or current() is attn_metadata: yield attn_metadata return - + attrs["attention_metadata"] = weakref.ref(attn_metadata) try: yield attn_metadata finally: - restore_attn_metadata_after_draft_replay(attn_metadata, saved_state) + attrs["attention_metadata"] = current + + @contextmanager + def draft_kv_cache_context(self, attn_metadata, draft_kv_cache_manager): + """The metadata a draft forward runs on: the draft metadata itself. + + ``md.draft`` is built over the draft KV cache (or the target's when + shared), so nothing is selected or swapped; while the context is open + the draft is the model's ``attention_metadata`` extra attribute. + """ + with self._as_attention_metadata(attn_metadata) as selected: + yield selected def _sample_sharded_temperature(self, logits: torch.Tensor, temperatures: torch.Tensor) -> torch.Tensor: diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index 50de723c3968..ce6d730916d8 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -460,84 +460,90 @@ def _forward_impl( spec_metadata=spec_metadata, attn_metadata=attn_metadata) - # update attn metadata - if attn_metadata is not None: - self.change_attn_metadata(num_accepted_tokens, attn_metadata, - spec_metadata) - - # Run MTP layers to predict draft tokens - next_draft_tokens = [] - last_tokens_idx = torch.cumsum( - attn_metadata.seq_lens_cuda, dim=0, dtype=torch.long) - 1 - - draft_kv_cache_manager = self.get_draft_kv_cache_manager( - resource_manager) - - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - for i, mtp_layer in enumerate( - draft_model.mtp_layers[:runtime_draft_len]): - if self.guided_decoder is not None: - new_tokens = draft_inputs['input_ids'][last_tokens_idx] - self.guided_decoder.add_draft_batch(new_tokens, - num_accepted_tokens, - draft_step=i) - - hidden_states = mtp_layer(embed_tokens=draft_model.embed_tokens, - **draft_inputs) - - logits = mtp_layer.shared_head(hidden_states, - draft_model.lm_head, - attn_metadata).float() - if self.guided_decoder is not None: - self.guided_decoder.execute_draft_batch(logits, + # The MTP layers run on the draft metadata, which leaves the target's + # untouched; without one, the target's state is saved and restored + # around them. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + self.change_attn_metadata(num_accepted_tokens, + draft_attn_metadata, + spec_metadata, + target=attn_metadata) + + # Run MTP layers to predict draft tokens + next_draft_tokens = [] + last_tokens_idx = torch.cumsum( + draft_attn_metadata.seq_lens_cuda, dim=0, dtype=torch.long) - 1 + + draft_kv_cache_manager = self.get_draft_kv_cache_manager( + resource_manager) + + with self.draft_kv_cache_context( + draft_attn_metadata, + draft_kv_cache_manager) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + draft_inputs["attn_metadata"] = draft_attn_metadata + for i, mtp_layer in enumerate( + draft_model.mtp_layers[:runtime_draft_len]): + if self.guided_decoder is not None: + new_tokens = draft_inputs['input_ids'][last_tokens_idx] + self.guided_decoder.add_draft_batch(new_tokens, + num_accepted_tokens, draft_step=i) - new_draft_token = self.sample_draft_tokens(logits, - spec_metadata, - batch_size, - draft_step=i) - next_draft_tokens.append(new_draft_token) - # shift input_ids and hidden_states - input_ids = draft_inputs["input_ids"] - input_ids[:-1] = input_ids[1:].clone() - input_ids[last_tokens_idx] = new_draft_token - draft_hidden_states = draft_inputs["hidden_states"] - draft_hidden_states[:-1] = draft_hidden_states[1:].clone() - draft_hidden_states[last_tokens_idx] = hidden_states[ - last_tokens_idx, :] - draft_inputs = { - "input_ids": input_ids, - "position_ids": draft_inputs["position_ids"], - "hidden_states": draft_hidden_states, - "attn_metadata": draft_inputs["attn_metadata"], - } - next_draft_tokens = torch.stack(next_draft_tokens, dim=1) - - # Override with SA draft tokens after all MTP layers have run, - # so that MTP layers never see SA tokens in their inputs. - if self.sa_enhancer is not None: - num_contexts = attn_metadata.num_contexts - gen_draft_tokens = next_draft_tokens[num_contexts:] - gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( - gen_draft_tokens) - next_draft_tokens[num_contexts:] = gen_draft_tokens - - # restore attn metadata - if attn_metadata is not None: - self._restore_attn_metadata_from_spec_dec(attn_metadata) - - # prepare next new tokens to support overlap scheduler - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, next_draft_tokens, - spec_metadata.batch_indices_cuda, batch_size, num_accepted_tokens) - - return { - 'logits': raw_logits, - 'new_tokens': accepted_tokens, - 'new_tokens_lens': num_accepted_tokens, - 'next_draft_tokens': next_draft_tokens, - 'next_new_tokens': next_new_tokens - } + hidden_states = mtp_layer( + embed_tokens=draft_model.embed_tokens, **draft_inputs) + + logits = mtp_layer.shared_head(hidden_states, + draft_model.lm_head, + draft_attn_metadata).float() + if self.guided_decoder is not None: + self.guided_decoder.execute_draft_batch(logits, + draft_step=i) + + new_draft_token = self.sample_draft_tokens(logits, + spec_metadata, + batch_size, + draft_step=i) + next_draft_tokens.append(new_draft_token) + # shift input_ids and hidden_states + input_ids = draft_inputs["input_ids"] + input_ids[:-1] = input_ids[1:].clone() + input_ids[last_tokens_idx] = new_draft_token + draft_hidden_states = draft_inputs["hidden_states"] + draft_hidden_states[:-1] = draft_hidden_states[1:].clone() + draft_hidden_states[last_tokens_idx] = hidden_states[ + last_tokens_idx, :] + draft_inputs = { + "input_ids": input_ids, + "position_ids": draft_inputs["position_ids"], + "hidden_states": draft_hidden_states, + "attn_metadata": draft_inputs["attn_metadata"], + } + next_draft_tokens = torch.stack(next_draft_tokens, dim=1) + + # Override with SA draft tokens after all MTP layers have run, + # so that MTP layers never see SA tokens in their inputs. + if self.sa_enhancer is not None: + num_contexts = attn_metadata.num_contexts + gen_draft_tokens = next_draft_tokens[num_contexts:] + gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( + gen_draft_tokens) + next_draft_tokens[num_contexts:] = gen_draft_tokens + + # prepare next new tokens to support overlap scheduler + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, next_draft_tokens, + spec_metadata.batch_indices_cuda, batch_size, + num_accepted_tokens) + + return { + 'logits': raw_logits, + 'new_tokens': accepted_tokens, + 'new_tokens_lens': num_accepted_tokens, + 'next_draft_tokens': next_draft_tokens, + 'next_new_tokens': next_new_tokens + } def skip_forward( self, @@ -923,43 +929,53 @@ def sample_and_accept_draft_tokens( def change_attn_metadata(self, num_accepted_tokens: torch.Tensor, attn_metadata: AttentionMetadata, - spec_metadata: MTPSpecMetadata): - self._prepare_attn_metadata_for_spec_dec(attn_metadata) + spec_metadata: MTPSpecMetadata, *, + target: AttentionMetadata): + """Set the MTP layers' lengths on ``attn_metadata``. + + Each generation request feeds ``runtime_draft_len`` tokens and keeps + only its accepted tokens' KV. ``attn_metadata`` is the draft metadata + (or the target's while its state is saved); the host cached-token + counts are derived from ``target``'s, which stay unchanged. + """ batch_size = attn_metadata.num_seqs runtime_draft_len = spec_metadata.runtime_draft_len num_contexts = attn_metadata.num_contexts - attn_metadata._seq_lens[num_contexts:batch_size] -= 1 - attn_metadata._seq_lens_cuda[num_contexts:batch_size] -= 1 - attn_metadata.on_update() + attn_metadata.adjust_seq_lens(slice(num_contexts, batch_size), -1) if hasattr(attn_metadata, 'kv_lens_cuda'): # Note that it's important to not free the seq_lens_cuda # buffer once the graph has been captured also - this will invalidate # the graph and force an expensive recapture. - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - runtime_draft_len + 1 - - num_accepted_tokens[num_contexts:batch_size]) # A generation request's KV length can never be smaller than the - # number of query tokens (mtp_num_modules) the draft layer processes - # this step. This only underflows for the tiny dummy sequences used - # during generation-step warmup; real sequences are long enough that - # subtracting the rejected draft tokens stays well above the draft - # length. Clamping in-place (CUDA-graph safe) keeps an invalid - # kv_len < q_len from reaching the attention kernel, which otherwise - # computes a negative effective KV range and triggers an illegal - # memory access (e.g. Step3p7 MTP with dense sliding-window attention). - attn_metadata.kv_lens_cuda[num_contexts:batch_size].clamp_( - min=runtime_draft_len) - attn_metadata.on_update_kv_lens() - if getattr(attn_metadata, "fp4_mla_state", None) is not None: - attn_metadata.update_for_spec_dec() - - if attn_metadata.kv_cache_params is not None and not attn_metadata.is_cuda_graph: - for i in range(num_contexts, batch_size): - # used for vanilla MLA, list on cpu - attn_metadata.kv_cache_params.num_cached_tokens_per_seq[ - i] -= runtime_draft_len + 1 - num_accepted_tokens[i].item() + # number of query tokens (mtp_num_modules) the draft layer + # processes this step. This only underflows for the tiny dummy + # sequences used during generation-step warmup; real sequences + # are long enough that subtracting the rejected draft tokens + # stays well above the draft length. Clamping in-place (CUDA-graph + # safe) keeps an invalid kv_len < q_len from reaching the + # attention kernel, which otherwise computes a negative effective + # KV range and triggers an illegal memory access (e.g. Step3p7 + # MTP with dense sliding-window attention). + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - + (runtime_draft_len + 1), + min_value=runtime_draft_len) + + target_params = target.kv_cache_params + if target_params is not None and not attn_metadata.is_cuda_graph: + # Host-side readers (vanilla MLA, host KV appends) see the + # accepted tokens only. + num_cached = list(target_params.num_cached_tokens_per_seq) + rejected = (runtime_draft_len + 1 - + num_accepted_tokens[num_contexts:batch_size]).tolist() + for i, num_rejected in enumerate(rejected, start=num_contexts): + num_cached[i] -= num_rejected + attn_metadata.set_cached_token_counts(num_cached) + # One refresh after every length change of the step. + self.refresh_draft() def prepare_drafter_inputs( self, diff --git a/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py b/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py index 271954edbe78..8de028148134 100644 --- a/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py +++ b/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py @@ -81,14 +81,6 @@ def __init__( self.spec_tree_manager = None self._d2t = None - # (all_rank_num_tokens, force_prepare_spec_dec_tree_mask) snapshot taken by - # _forward_impl before mutating attn_metadata; None means "not saved / already - # restored" (all_rank_num_tokens is legitimately None outside ADP, so we can't - # use it directly as a sentinel). Restored by _ensure_spec_dec_state_restored - # on the failure path so a tolerated forward exception does not leak state - # into subsequent iterations. - self._saved_extra_state = None - # Pre-allocated draft-loop buffers (CUDA-graph safe). self.draft_tokens_buffer = torch.zeros( max_batch_size, loop_max_tokens, dtype=torch.int32, device="cuda" @@ -177,78 +169,11 @@ def __init__( sm = get_sm_version() self._needs_mask_repack = not is_sm_100f(sm) - def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): - super()._prepare_attn_metadata_for_spec_dec(attn_metadata) - - batch_size = attn_metadata.num_seqs - if hasattr(attn_metadata, "kv_lens_cuda"): - # Keep kv_lens_cuda itself alive because TRTLLM attention holds a - # runtime view into it. - self._saved_kv_lens_cuda = attn_metadata.kv_lens_cuda[:batch_size].clone() - else: - self._saved_kv_lens_cuda = None - - # Restore verify metadata after the draft loop mutates it. - if attn_metadata.spec_decoding_packed_mask is not None: - self._saved_packed_mask = attn_metadata.spec_decoding_packed_mask[:batch_size].clone() - else: - self._saved_packed_mask = None - if attn_metadata.spec_decoding_position_offsets is not None: - self._saved_position_offsets = attn_metadata.spec_decoding_position_offsets.clone() - self._saved_spec_decoding_query_len = attn_metadata.spec_decoding_query_len - else: - self._saved_position_offsets = None - self._saved_spec_decoding_query_len = None - if attn_metadata.spec_decoding_generation_lengths is not None: - self._saved_generation_lengths = attn_metadata.spec_decoding_generation_lengths[ - :batch_size - ].clone() - else: - self._saved_generation_lengths = None - def prepare_position_ids_and_last_tokens(self, position_ids, seq_lens_cuda): position_ids = position_ids.squeeze(0) last_tokens_idx = torch.cumsum(seq_lens_cuda, dim=0, dtype=torch.long) - 1 return position_ids, last_tokens_idx - def _restore_attn_metadata_from_spec_dec(self, attn_metadata): - super()._restore_attn_metadata_from_spec_dec(attn_metadata) - - if self._saved_kv_lens_cuda is not None: - batch_size = self._saved_kv_lens_cuda.shape[0] - attn_metadata.kv_lens_cuda[:batch_size].copy_(self._saved_kv_lens_cuda) - self._saved_kv_lens_cuda = None - - if self._saved_packed_mask is not None: - batch_size = self._saved_packed_mask.shape[0] - attn_metadata.spec_decoding_packed_mask[:batch_size].copy_(self._saved_packed_mask) - self._saved_packed_mask = None - if self._saved_position_offsets is not None: - attn_metadata.spec_decoding_position_offsets.copy_(self._saved_position_offsets) - attn_metadata.spec_decoding_query_len = self._saved_spec_decoding_query_len - self._saved_position_offsets = None - self._saved_spec_decoding_query_len = None - if self._saved_generation_lengths is not None: - batch_size = self._saved_generation_lengths.shape[0] - attn_metadata.spec_decoding_generation_lengths[:batch_size].copy_( - self._saved_generation_lengths - ) - self._saved_generation_lengths = None - - def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - # Base cleanup handles the has_spec_dec_saved_state path. Undo the - # dynamic-tree-only mutations (all_rank_num_tokens, - # force_prepare_spec_dec_tree_mask) so a tolerated forward failure - # does not leak the pre-draft-loop state into later iterations. - super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - if attn_metadata is None or self._saved_extra_state is None: - return - ( - attn_metadata.all_rank_num_tokens, - attn_metadata.force_prepare_spec_dec_tree_mask, - ) = self._saved_extra_state - self._saved_extra_state = None - # ------------------------------------------------------------------ # # Helpers # # ------------------------------------------------------------------ # @@ -710,56 +635,47 @@ def _forward_impl( # Save attn/spec metadata before the draft loop mutates it. Stashed on # self so _ensure_spec_dec_state_restored can undo the mutation if the # draft loop raises (see SpecWorkerBase.forward try/finally). - self._saved_extra_state = ( - attn_metadata.all_rank_num_tokens, - attn_metadata.force_prepare_spec_dec_tree_mask, - ) - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - attn_metadata.force_prepare_spec_dec_tree_mask = True + # The draft loop runs on the draft metadata, which leaves the target's + # untouched; without one, the target's state is saved and restored. + with self.draft_loop(attn_metadata) as draft_attn_metadata: + draft_attn_metadata.force_prepare_spec_dec_tree_mask = True + + # (c) Run the MTP draft tree loop -> build + store the next tree. + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + next_draft_tokens = self._forward_draft_loop( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + draft_kv_cache_manager=draft_kv_cache_manager, + num_contexts=num_contexts, + num_gens=num_gens, + batch_size=batch_size, + ) - # (c) Run the MTP draft tree loop -> build + store the next tree. - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - next_draft_tokens = self._forward_draft_loop( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - draft_kv_cache_manager=draft_kv_cache_manager, - num_contexts=num_contexts, - num_gens=num_gens, - batch_size=batch_size, - ) + attn_metadata.use_spec_decoding = True - # Restore attn metadata to support cuda graph. - self._restore_attn_metadata_from_spec_dec(attn_metadata) - ( - attn_metadata.all_rank_num_tokens, - attn_metadata.force_prepare_spec_dec_tree_mask, - ) = self._saved_extra_state - self._saved_extra_state = None - attn_metadata.use_spec_decoding = True - - # (d) Prepare next_new_tokens for overlap scheduler. - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + # (d) Prepare next_new_tokens for overlap scheduler. + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - "accepted_draft_tokens_indices": self._accepted_draft_indices_tensor[:batch_size], - } + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + "accepted_draft_tokens_indices": self._accepted_draft_indices_tensor[:batch_size], + } # ------------------------------------------------------------------ # # Step-0 drafter-input repack (dynamic tree) # @@ -843,9 +759,12 @@ def _prepare_step0_drafter_inputs( dim=0, ) - attn_metadata._seq_lens[num_contexts:batch_size].fill_(max_path_len) - attn_metadata._seq_lens_cuda[num_contexts:batch_size].fill_(max_path_len) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(num_contexts, batch_size), + host=max_path_len, + device=max_path_len, + update_counts=True, + ) else: # Context-only (warmup): no gen tokens to repack. input_ids = input_ids_ctx @@ -914,13 +833,19 @@ def _forward_draft_loop( self._repack_mask_padded_to_packed(packed_mask, num_gens, num_step0_tokens) attn_metadata.use_spec_decoding = num_gens > 0 if num_gens > 0 and hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= self._kv_correction + attn_metadata.adjust_kv_lens(slice(num_contexts, batch_size), -self._kv_correction) self._refresh_blackwell_tree_mask_metadata(attn_metadata) + self.refresh_draft() if spec_metadata.all_rank_num_tokens is not None: # Keep attention/MoE token counts aligned with step-0 repack. attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_tokens - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata hidden_states = draft_model.mtp_layers[0]( embed_tokens=draft_model.embed_tokens, all_rank_num_tokens=spec_metadata.all_rank_num_tokens, @@ -933,7 +858,7 @@ def _forward_draft_loop( step0_hs = hidden_states[gather_ids] logits = draft_model.mtp_layers[0].shared_head( - step0_hs, draft_model.lm_head, attn_metadata, True + step0_hs, draft_model.lm_head, draft_attn_metadata, True ) new_draft_tokens, new_draft_scores = self.sample( @@ -945,12 +870,12 @@ def _forward_draft_loop( new_draft_scores=new_draft_scores, previous_draft_scores=None, batch_size=batch_size, - attn_metadata=attn_metadata, + attn_metadata=draft_attn_metadata, ) self.update_hidden_states(cur_draft_idx=0, batch_size=batch_size, step0_hs=step0_hs) self._prepare_draft_layer_metadata( 0, - attn_metadata, + draft_attn_metadata, batch_size, gather_ids, num_contexts, @@ -969,7 +894,7 @@ def _forward_draft_loop( subseq_all_rank_num_tokens = [ n * num_tokens_per_req for n in spec_metadata.all_rank_num_seqs ] - attn_metadata.all_rank_num_tokens = subseq_all_rank_num_tokens + draft_attn_metadata.all_rank_num_tokens = subseq_all_rank_num_tokens inp_hs = self._accumulated_hs[:batch_size, :num_tokens_per_req, :].reshape( num_infer_tokens, -1 @@ -980,7 +905,7 @@ def _forward_draft_loop( "input_ids": inp_ids, "position_ids": inp_pos, "hidden_states": inp_hs, - "attn_metadata": attn_metadata, + "attn_metadata": draft_attn_metadata, } hidden_states = draft_model.mtp_layers[0]( embed_tokens=draft_model.embed_tokens, @@ -993,7 +918,7 @@ def _forward_draft_loop( hs_reshaped = hidden_states.reshape(batch_size, num_tokens_per_req, -1) selected_hs = hs_reshaped[:, -self.K :, :].reshape(batch_size * self.K, -1) logits = draft_model.mtp_layers[0].shared_head( - selected_hs, draft_model.lm_head, attn_metadata, True + selected_hs, draft_model.lm_head, draft_attn_metadata, True ) new_draft_tokens, new_draft_scores = self.sample( @@ -1008,7 +933,7 @@ def _forward_draft_loop( new_draft_scores=new_draft_scores, previous_draft_scores=previous_draft_scores, batch_size=batch_size, - attn_metadata=attn_metadata, + attn_metadata=draft_attn_metadata, ) self.update_hidden_states( cur_draft_idx=layer_idx, @@ -1016,7 +941,7 @@ def _forward_draft_loop( hidden_states_to_save=hidden_states, selected_parents=self._last_selected_parents, ) - self._prepare_draft_layer_metadata(layer_idx, attn_metadata, batch_size) + self._prepare_draft_layer_metadata(layer_idx, draft_attn_metadata, batch_size) # Resample the final tree and build it into slot_storage. real_draft_tokens, topk_score_indices = self.resampling_final_draft_tokens(batch_size) @@ -1058,23 +983,28 @@ def _prepare_draft_layer_metadata( -1, self.K ) - attn_metadata._seq_lens[:batch_size].fill_(self.K) - attn_metadata._seq_lens_cuda[:batch_size].fill_(self.K) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(0, batch_size), + host=self.K, + device=self.K, + update_counts=True, + ) if inputs["attn_metadata"].kv_cache_manager is not None: - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) - attn_metadata.num_contexts = 0 + # The context requests continue as generation rows. + attn_metadata.as_generation_rows() if hasattr(attn_metadata, "kv_lens_cuda"): # Rewind only unaccepted verify tokens; draft KV is added later. if num_gens > 0: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - self._max_path_len - ) - num_accepted_tokens[num_contexts:batch_size] - attn_metadata.kv_lens_cuda[:batch_size] += self.K + attn_metadata.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - self._max_path_len, + ) + attn_metadata.adjust_kv_lens(slice(0, batch_size), self.K) attn_metadata.use_spec_decoding = True self._refresh_blackwell_tree_mask_metadata(attn_metadata) + self.refresh_draft() else: num_tokens_previous_layer = cur_draft_idx * self.K num_tokens_current_layer = self.K * (cur_draft_idx + 1) @@ -1082,12 +1012,16 @@ def _prepare_draft_layer_metadata( self.position_ids_buffer[ :batch_size, num_tokens_previous_layer:num_tokens_current_layer ] = prev_pos[:, -self.K :] + 1 - attn_metadata._seq_lens[:batch_size].fill_(num_tokens_current_layer) - attn_metadata._seq_lens_cuda[:batch_size].fill_(num_tokens_current_layer) - attn_metadata.on_update() + attn_metadata.set_seq_lens( + slice(0, batch_size), + host=num_tokens_current_layer, + device=num_tokens_current_layer, + update_counts=True, + ) if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[:batch_size] += self.K + attn_metadata.adjust_kv_lens(slice(0, batch_size), self.K) self._refresh_blackwell_tree_mask_metadata(attn_metadata) + self.refresh_draft() class MTPEagleDynamicTreeResourceManager(BaseResourceManager): diff --git a/tensorrt_llm/_torch/speculative/pard.py b/tensorrt_llm/_torch/speculative/pard.py index 9863e788fde8..37c52d396e04 100644 --- a/tensorrt_llm/_torch/speculative/pard.py +++ b/tensorrt_llm/_torch/speculative/pard.py @@ -94,12 +94,6 @@ def __init__( self.sa_enhancer: Optional[SADraftEnhancer] = None if getattr(spec_config, "sa_config", None) is not None: self.sa_enhancer = SADraftEnhancer(spec_config.sa_config.threshold) - # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, - # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). - self._kv_rewind_pending = False - self._kv_rewind_amount = None - self._kv_rewind_nc = None - self._kv_rewind_bs = None logger.info( f"PARDWorker initialized with use_separate_draft_kv_cache={use_separate_draft_kv_cache}" ) @@ -108,31 +102,14 @@ def __init__( def max_draft_len(self) -> int: return self.spec_config.max_draft_len - def _prepare_attn_metadata_for_pard(self, attn_metadata, spec_metadata): - """ - Save attn_metadata fields that PARD modifies during forward. - - CUDA graph handling: During warmup, save and restore kv_lens_cuda to - prevent accumulation. During capture, don't save kv_lens_cuda so the - modification is captured but restoration is not — ensuring the - modification persists correctly during replay. - """ - is_capturing = torch.cuda.is_current_stream_capturing() - - if spec_metadata.is_cuda_graph and not is_capturing: - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda", "kv_lens_cuda") - else: - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - def _prepare_kv_for_draft_forward( self, attn_metadata, - num_accepted_tokens: torch.Tensor, num_contexts: int, batch_size: int, ): """ - Adjust kv_lens_cuda so the draft model sees correct RoPE positions. + Adjust the draft's KV lengths so the draft model sees correct RoPE positions. The attention kernel computes positions as ``position_i = kv_lens - seq_lens + i``. The batch manager sets @@ -140,55 +117,13 @@ def _prepare_kv_for_draft_forward( giving ``position_0 = past_seen``. The target's bonus token sits at past_seen, so the draft's first accepted token (acc_0) needs position past_seen + 1. We add 1 to kv_lens_cuda to achieve this. - - The corresponding rewind is deferred to _apply_kv_rewind_after_draft. + ``attn_metadata`` is the draft metadata, which the next draft loop + starts again from the target's lengths. """ if hasattr(attn_metadata, "kv_lens_cuda"): - # Rewind amount: after draft forward, subtract (1 - num_accepted) - # so kv_lens lands at past_seen + 2K + num_accepted for the next - # target iteration. - self._kv_rewind_amount = 1 - num_accepted_tokens[num_contexts:batch_size] - self._kv_rewind_nc = num_contexts - self._kv_rewind_bs = batch_size - if batch_size > num_contexts: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 - self._kv_rewind_pending = True - - attn_metadata.update_for_spec_dec() - - def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): - """ - Apply the deferred kv_lens rewind after the draft forward. - - Skipped during CUDA graph warmup (where kv_lens_cuda is saved/restored - by prepare_for_spec_dec) to avoid cumulative shrinkage. Applied during - capture and normal inference. - """ - self._kv_rewind_pending = False - is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() - if is_warmup: - # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and - # is restored wholesale, so no rewind is needed. - return - - if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): - nc = self._kv_rewind_nc - bs = self._kv_rewind_bs - attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount - attn_metadata.kv_lens_cuda[nc:bs].clamp_(min=0) - - def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - # Restore first (in warmup mode kv_lens_cuda was saved and comes back - # wholesale), then apply any pending rewind for the other modes so a - # failed draft forward does not leave kv_lens_cuda incremented. - super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - if ( - getattr(self, "_kv_rewind_pending", False) - and attn_metadata is not None - and spec_metadata is not None - ): - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + attn_metadata.adjust_kv_lens(slice(num_contexts, batch_size), 1) + self.refresh_draft() def _forward_impl( self, @@ -244,122 +179,136 @@ def _forward_impl( max_draft_len=K, ) - self._prepare_attn_metadata_for_pard(attn_metadata, spec_metadata) - self._prepare_kv_for_draft_forward( - attn_metadata, num_accepted_tokens, num_contexts, batch_size - ) - - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - ) + # The draft forward runs on the draft metadata, which leaves the + # target's untouched; without one, the target's lengths are saved and + # restored around it. + target_attn_metadata = attn_metadata + with self.draft_loop(target_attn_metadata) as attn_metadata: + self._prepare_kv_for_draft_forward(attn_metadata, num_contexts, batch_size) + + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + + if num_gens > 0: + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata + hidden_states_out = draft_model.model(**inputs) + + # Gather K logits per gen request starting at the bonus position. + # Layout: [acc_0..acc_M, masks] (2K total). Positions M..M+K-1 + # produce K unique draft predictions (bonus + K-1 masks). + gen_start_idx = draft_attn_metadata.num_ctx_tokens + + request_bases = ( + torch.arange(num_gens, dtype=torch.long, device="cuda") * (2 * K) + + gen_start_idx + ) - if num_gens > 0: - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - hidden_states_out = draft_model.model(**inputs) + gen_num_accepted = num_accepted_tokens[num_contexts:batch_size].long() + base_offsets = gen_num_accepted - 1 # M = bonus position + offsets = torch.arange(K, dtype=torch.long, device="cuda") + + gen_gather_ids = ( + request_bases.unsqueeze(1) + + base_offsets.unsqueeze(1) + + offsets.unsqueeze(0) + ).flatten() + gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) + + gen_logits = draft_model.logits_processor( + hidden_states_out[gen_gather_ids], + draft_model.lm_head, + draft_attn_metadata, + True, + ) - # Gather K logits per gen request starting at the bonus position. - # Layout: [acc_0..acc_M, masks] (2K total). Positions M..M+K-1 - # produce K unique draft predictions (bonus + K-1 masks). - gen_start_idx = attn_metadata.num_ctx_tokens + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - request_bases = ( - torch.arange(num_gens, dtype=torch.long, device="cuda") * (2 * K) - + gen_start_idx - ) + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, + spec_metadata, + batch_size, + num_contexts=num_contexts, + ) - gen_num_accepted = num_accepted_tokens[num_contexts:batch_size].long() - base_offsets = gen_num_accepted - 1 # M = bonus position - offsets = torch.arange(K, dtype=torch.long, device="cuda") + if self.sa_enhancer is not None and sa_manager is not None: + gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( + gen_draft_tokens + ) + + # Pad from (num_gens, K) to (num_gens, 2K-1). + if K > 1: + pad = torch.zeros((num_gens, K - 1), dtype=torch.int32, device="cuda") + gen_draft_tokens = torch.cat([gen_draft_tokens, pad], dim=1) + + elif num_contexts > 0 and self.use_separate_draft_kv_cache: + # Pure context batch: populate the draft KV cache so it's + # ready when generation starts. + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata + draft_model.model(**inputs) + gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") - gen_gather_ids = ( - request_bases.unsqueeze(1) + base_offsets.unsqueeze(1) + offsets.unsqueeze(0) - ).flatten() - gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) + else: + gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs( + spec_metadata, num_contexts, num_gens, K, gen_vocab + ) - gen_logits = draft_model.logits_processor( - hidden_states_out[gen_gather_ids], draft_model.lm_head, attn_metadata, True + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros( + (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" ) - - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros( + (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" ) + else: + next_draft_tokens = gen_draft_tokens - if self.sa_enhancer is not None and sa_manager is not None: - gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( - gen_draft_tokens - ) - - # Pad from (num_gens, K) to (num_gens, 2K-1). - if K > 1: - pad = torch.zeros((num_gens, K - 1), dtype=torch.int32, device="cuda") - gen_draft_tokens = torch.cat([gen_draft_tokens, pad], dim=1) - - elif num_contexts > 0 and self.use_separate_draft_kv_cache: - # Pure context batch: populate the draft KV cache so it's - # ready when generation starts. - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - draft_model.model(**inputs) - gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") - - else: - gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros( - (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" - ) - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros( - (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, ) - else: - next_draft_tokens = gen_draft_tokens - - self._restore_attn_metadata_from_spec_dec(attn_metadata) - - # Deferred kv_lens rewind (must happen after restore so it persists). - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) - self._rollback_guided_decoder_after_verify(num_accepted_tokens) - - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) - - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def _reshape_draft_tokens_for_accept(self, spec_metadata, num_gens, device): # draft_tokens buffer has (2K-1) entries per gen request; extract the K real drafts. @@ -457,10 +406,12 @@ def prepare_1st_drafter_inputs( input_ids_gen = request_ids_2d.flatten() # Update seq_lens for gen requests to 2K - attn_metadata._seq_lens_cuda[num_contexts : num_contexts + num_gens] = ( - total_tokens_per_req + attn_metadata.set_seq_lens( + slice(num_contexts, num_contexts + num_gens), + host=total_tokens_per_req, + device=total_tokens_per_req, ) - attn_metadata._seq_lens[num_contexts : num_contexts + num_gens] = total_tokens_per_req + self.refresh_draft() # Position IDs: base = kv_lens - 2K = past_seen (correct for acc_0). # With TrtllmAttention (rope_fusion=True) these are ignored in diff --git a/tensorrt_llm/tools/layer_wise_benchmarks/runner.py b/tensorrt_llm/tools/layer_wise_benchmarks/runner.py index ef7f7dd25fe8..ca57b7aca9ae 100644 --- a/tensorrt_llm/tools/layer_wise_benchmarks/runner.py +++ b/tensorrt_llm/tools/layer_wise_benchmarks/runner.py @@ -16,6 +16,8 @@ import tensorrt_llm._torch.model_config import tensorrt_llm._torch.pyexecutor.config_utils import tensorrt_llm.bindings +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig +from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata from tensorrt_llm._torch.attention.backends.utils import get_attention_backend from tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops import GroupedGemmInputsHelper from tensorrt_llm._torch.metadata import KVCacheParams @@ -706,6 +708,21 @@ def create_run_pack( if sparse_attention_config is not None else None ) + # seq_len_q > 1 means MTP: each request submits 1 + num_draft tokens. + # The DSA indexer sizes its buffers from the draft length; without it + # its context_lens buffer stays one column wide and DeepGEMM aborts on + # the next_n mismatch. Shapes only -- spec-dec masking stays off. Only + # DSA needs it: max_total_draft_tokens otherwise reaches the attention + # op cache key and FMHA kernel selection. + spec_settings = {} + if ( + run_type == "GEN" + and seq_len_q > 1 + and any(issubclass(cls, DSAMetadata) for cls in metadata_cls.backend_extension_types) + ): + spec_settings["spec_decoding"] = SpecDecodingConfig( + max_draft_len=seq_len_q - 1, max_total_draft_tokens=seq_len_q - 1 + ) attn_metadata = metadata_cls( seq_lens=torch.tensor([seq_len_q] * batch_size, dtype=torch.int), request_ids=list(range(request_id_begin, request_id_begin + batch_size)), @@ -730,32 +747,13 @@ def create_run_pack( workspace=attn_workspace, mapping=self.model_config.mapping, sparse_metadata_params=sparse_metadata_params, + **spec_settings, ) # One entry per attention-DP rank, not per world rank: the non-DP MoE path # asserts len(all_rank_num_tokens) == 1. attn_metadata.all_rank_num_tokens = [ batch_size * seq_len_q ] * self.model_config.mapping.dp_size - # seq_len_q > 1 means MTP: each request submits 1 + num_draft tokens. In - # serving the executor announces that via update_spec_dec_param(), the only - # place max_draft_tokens is set. Without it the DSA indexer's context_lens - # buffer stays one column wide and DeepGEMM aborts on the next_n mismatch. - # Shapes only -- spec-dec masking stays off. - # - # Gate on kv_lens_cuda_2d, not on the method: update_spec_dec_param() is on - # the base metadata class, so hasattr() would let every backend in, and the - # base sets max_total_draft_tokens unconditionally -- which reaches the - # attention op cache key and FMHA kernel selection. kv_lens_cuda_2d exists - # only on DSA metadata, which is the backend that needs this. - if run_type == "GEN" and seq_len_q > 1 and hasattr(attn_metadata, "kv_lens_cuda_2d"): - attn_metadata.update_spec_dec_param( - batch_size=batch_size, - is_spec_decoding_enabled=False, - is_spec_dec_tree=False, - is_spec_dec_dynamic_tree=False, - max_draft_len=seq_len_q - 1, - max_total_draft_tokens=seq_len_q - 1, - ) attn_metadata.prepare() hidden_size = pretrained_config.hidden_size position_ids = torch.tensor( diff --git a/tests/microbenchmarks/attention_perf/attention_perf_harness.py b/tests/microbenchmarks/attention_perf/attention_perf_harness.py index bc7ccc641f46..d55893949445 100644 --- a/tests/microbenchmarks/attention_perf/attention_perf_harness.py +++ b/tests/microbenchmarks/attention_perf/attention_perf_harness.py @@ -52,6 +52,7 @@ import tensorrt_llm from tensorrt_llm._torch.attention.backends import TrtllmAttention +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionInputType, AttentionRuntimeFeatures, @@ -548,7 +549,7 @@ def _build_mla_gen_metadata(case: AttnCase, kv_mgr: KVCacheManagerV2): mapping=Mapping(world_size=1, tp_size=1, rank=0), # FlashMLA decode kernel is sm90-only — that is where the DeepSeek-R1 # gen regressions live; on other archs this routes to the generic kernel. - enable_flash_mla=(torch.cuda.get_device_capability() == (9, 0)), + extension_types=(FlashMlaMetadata,) if torch.cuda.get_device_capability() == (9, 0) else (), ) metadata = TrtllmAttention.Metadata(**kwargs) metadata.prepare() @@ -734,7 +735,7 @@ def _build_dsa_ctx(case: AttnCase, device: torch.device): use_cache=True, num_cached_tokens_per_seq=[0] * case.batch_size ), mapping=Mapping(world_size=1, tp_size=1, rank=0), - sparse_attention_config=sparse_config, + sparse_metadata_params=sparse_config.to_sparse_metadata_params(), ) metadata.prepare() diff --git a/tests/microbenchmarks/bench_fp4_mla_decode.py b/tests/microbenchmarks/bench_fp4_mla_decode.py index 32466a6c50ca..c51d286a8c17 100644 --- a/tests/microbenchmarks/bench_fp4_mla_decode.py +++ b/tests/microbenchmarks/bench_fp4_mla_decode.py @@ -137,6 +137,13 @@ def _expand_qk_residual_terms( ) +def _fp4_mla_metadata(**fields) -> SimpleNamespace: + """A metadata stub whose ``fp4_mla_state`` is also its FP4 MLA extension.""" + metadata = SimpleNamespace(**fields) + metadata.find_ext = lambda cls: metadata.fp4_mla_state if cls is Fp4MlaState else None + return metadata + + def _build_multi_seq_metadata( kv_cache_manager: Fp4MlaKVCacheManagerV2, *, seq_lens: list[int], page_size: int, layer_idx: int ) -> SimpleNamespace: @@ -194,7 +201,7 @@ def _build_multi_seq_metadata( kv_global_scale = torch.tensor([_BENCH_KV_GLOBAL_SCALE], dtype=torch.float32, device=device) q_global_scale = torch.tensor([FP4_MLA_Q_GLOBAL_SCALE], dtype=torch.float32, device=device) - return SimpleNamespace( + return _fp4_mla_metadata( kv_cache_manager=kv_cache_manager, page_size=page_size, num_contexts=num_seqs, diff --git a/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py b/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py new file mode 100644 index 000000000000..7af36ba725c0 --- /dev/null +++ b/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py @@ -0,0 +1,812 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU coverage of the metadata extension naming and attribute contracts.""" + +import copy +import inspect +from types import SimpleNamespace + +import pytest +import torch +from _torch.helpers import refresh_from_lengths, start_draft_loop + +from tensorrt_llm._torch.attention.backends.extensions import ( + MetadataExtension, + reserved_extension_names, +) +from tensorrt_llm._torch.attention.backends.features import ( + HelixMetadata, + RecurrentConfig, + RecurrentMetadata, +) +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata, BatchInputs +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata +from tensorrt_llm._torch.attention.backends.sparse.params import ( + MTPIndexShareExtension, + MTPIndexShareUser, + SparseAlgorithmExtension, +) +from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata +from tensorrt_llm._torch.memory_buffer_utils import Buffers +from tensorrt_llm._torch.metadata import KVCacheParams + +pytestmark = pytest.mark.cpu_only + + +def _metadata(*extension_types) -> AttentionMetadata: + return AttentionMetadata(max_num_requests=1, max_num_tokens=1, extension_types=extension_types) + + +def test_undeclared_metadata_attribute_is_rejected(): + metadata = _metadata() + with pytest.raises(AttributeError, match="csa2_remote_tail_mode"): + metadata.csa2_remote_tail_mode = "source" + + +def test_forwarded_fields_write_through_to_their_owner(): + metadata = _metadata(_Probe) + metadata.all_rank_num_tokens = [1] + metadata.probe_value = 1 + assert metadata.core().all_rank_num_tokens == [1] + assert metadata.ext(_Probe).probe_value == 1 + metadata._num_tokens = 3 + assert metadata.core()._num_tokens == 3 + + +def test_metadata_built_without_init_routes_core_fields_to_its_core(): + metadata = object.__new__(AttentionMetadata) + assert metadata.request_ids is None + metadata.request_ids = [0] + assert metadata.core().request_ids == [0] + with pytest.raises(AttributeError, match="ad_hoc"): + metadata.ad_hoc = 1 + + +def test_extension_cannot_declare_a_reserved_name(): + assert "prepare" in reserved_extension_names() + with pytest.raises(TypeError, match="reserved"): + + class _Reserved(MetadataExtension): + methods = frozenset({"prepare"}) + + +def test_two_extension_classes_cannot_declare_one_name(): + with pytest.raises(TypeError, match="already declares"): + + class _Shadow(MetadataExtension): + fields = frozenset({"helix_position_offsets"}) + + +def test_subclass_may_redeclare_its_base_names(): + class _Derived(HelixMetadata): + fields = frozenset({"helix_position_offsets"}) + + assert "helix_position_offsets" in _Derived.declared_fields() + + +def test_extension_cannot_shadow_a_core_attribute(): + class _Core(MetadataExtension): + fields = frozenset({"_core_shadow_probe"}) + + class _Metadata(AttentionMetadata): + _instance_attrs = frozenset({"_core_shadow_probe"}) + + with pytest.raises(ValueError, match="already defines"): + _Metadata(max_num_requests=1, max_num_tokens=1, extension_types=(_Core,)) + + +def test_extensions_with_shared_bases_attach_together(): + assert not HelixMetadata.declared_names() & SpecDecodingMetadata.declared_names() + shared_bases = MTPIndexShareUser.declared_names() | SparseAlgorithmExtension.declared_names() + assert not DSAMetadata.declared_names() & QSAMetadata.declared_names() - shared_bases + + +def test_index_share_owns_the_shared_draft_loop_state(): + assert MTPIndexShareExtension in DSAMetadata.requires + assert MTPIndexShareExtension in QSAMetadata.requires + shared = MTPIndexShareExtension.declared_fields() + assert not shared & DSAMetadata.declared_fields() + assert not shared & QSAMetadata.declared_fields() + + +needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="seq_lens are staged to CUDA") + + +@needs_cuda +def test_prepare_applies_batch_inputs(): + metadata = _metadata() + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor([3, 1], dtype=torch.int32), + num_contexts=1, + request_ids=[7, 8], + prompt_lens=[3, 5], + beam_width=1, + num_chunked_ctx_requests=0, + ) + ) + assert metadata.num_contexts == 1 + assert metadata.num_generations == 1 + assert metadata.num_tokens == 4 + assert metadata.request_ids == [7, 8] + assert metadata.cu_q_seqlens is None + + +@needs_cuda +def test_metadata_without_explicit_offsets_rejects_gapped_offsets(): + inputs = dict( + seq_lens=torch.tensor([3, 1], dtype=torch.int32), + num_contexts=2, + request_ids=[0, 1], + prompt_lens=[3, 1], + ) + metadata = _metadata() + assert not metadata.supports_explicit_offsets + metadata.prepare(BatchInputs(**inputs, q_offsets=torch.tensor([0, 3, 4], dtype=torch.int32))) + with pytest.raises(ValueError, match="cu_q_seqlens"): + metadata.prepare( + BatchInputs(**inputs, q_offsets=torch.tensor([0, 4, 6], dtype=torch.int32)) + ) + + +@needs_cuda +def test_length_mutation_apis_keep_buffers_and_counts(): + metadata = _metadata() + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor([2, 4, 4], dtype=torch.int32), + num_contexts=1, + request_ids=[0, 1, 2], + prompt_lens=[2, 9, 9], + ) + ) + host, device = metadata.seq_lens, metadata.seq_lens_cuda + if device.is_cuda: + # prepare copies the pinned host lengths to the device asynchronously; + # the host edits below must not race that copy. + torch.cuda.current_stream().synchronize() + metadata.adjust_seq_lens(slice(1, 3), -1) + assert metadata.seq_lens is host and metadata.seq_lens_cuda is device + assert host.tolist() == [2, 3, 3] and device.tolist() == [2, 3, 3] + assert metadata.num_tokens == 8 + + metadata.set_seq_lens(slice(1, 3), host=5, device=5) + assert host.tolist() == [2, 5, 5] and device.tolist() == [2, 5, 5] + assert metadata.num_tokens == 8 + metadata.set_seq_lens(slice(1, 3), host=1, device=1, update_counts=True) + assert metadata.num_tokens == 4 + + +def test_recurrent_settings_come_from_construction(): + metadata = SimpleNamespace(cuda_graph_buffers=None) + assert RecurrentMetadata(metadata).chunk_size == 128 + recurrent = RecurrentMetadata(metadata, recurrent_config=RecurrentConfig(chunk_size=64)) + assert recurrent.chunk_size == 64 + assert recurrent.mamba_metadata_cls is None + + +class _Probe(MetadataExtension): + fields = frozenset({"probe_value"}) + methods = frozenset({"probe"}) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.probe_value = None + + def probe(self) -> str: + return "probe" + + +def test_composed_class_forwards_extension_fields_and_methods(): + metadata = _metadata(_Probe) + probe = metadata.ext(_Probe) + assert isinstance(metadata, AttentionMetadata) + assert metadata.core_class() is AttentionMetadata + assert type(metadata) is not AttentionMetadata + + marker = object() + probe.probe_value = marker + assert probe.probe_value is marker + assert metadata.probe_value is marker + assert metadata.probe() == "probe" + + # Without the extension, the name is not an attribute of the metadata. + plain = _metadata() + assert not hasattr(plain, "probe_value") + with pytest.raises(AttributeError): + plain.probe_value = marker + + +def test_composed_class_is_shared_and_survives_copies(): + first, second = _metadata(_Probe), _metadata(_Probe) + assert type(first) is type(second) + clone = copy.copy(first) + assert type(clone) is type(first) + + +def test_extension_mutation_recomposes(): + metadata = _metadata() + metadata.add_ext(_Probe) + assert metadata.ext(_Probe).probe_value is metadata.probe_value + metadata.remove_ext(_Probe) + assert not hasattr(metadata, "probe_value") + + +class _TensorProbe(MetadataExtension): + fields = frozenset({"probe_scale"}) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.probe_scale = torch.full((1,), 2.0) + + +def _prepared_probe_metadata(seq_lens) -> AttentionMetadata: + metadata = _metadata(_TensorProbe) + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + num_contexts=1, + request_ids=list(range(len(seq_lens))), + prompt_lens=list(seq_lens), + ) + ) + return metadata + + +def _read_forwarded(x: torch.Tensor, metadata: AttentionMetadata) -> torch.Tensor: + out = x * metadata.num_tokens + metadata.num_contexts + out = out + metadata.seq_lens.sum() + return out * metadata.probe_scale + + +@needs_cuda +def test_dynamo_traces_forwarded_fields_without_breaks_or_recompiles(): + from torch._dynamo.testing import CompileCounter + + torch._dynamo.reset() + first = _prepared_probe_metadata([3, 1]) + x = torch.ones(4) + expected = _read_forwarded(x, first) + + explanation = torch._dynamo.explain(_read_forwarded)(x, first) + assert explanation.graph_break_count == 0, explanation.break_reasons + + fullgraph = torch.compile(_read_forwarded, backend="eager", fullgraph=True) + torch.testing.assert_close(fullgraph(x, first), expected) + + torch._dynamo.reset() + counter = CompileCounter() + compiled = torch.compile(_read_forwarded, backend=counter) + torch.testing.assert_close(compiled(x, first), expected) + # Another metadata of the same composed class and batch shape reuses the + # compiled frame; a changed extension tensor value does not recompile. + second = _prepared_probe_metadata([3, 1]) + second.ext(_TensorProbe).probe_scale = torch.full((1,), 3.0) + torch.testing.assert_close(compiled(x, second), _read_forwarded(x, second)) + assert counter.frame_count == 1 + + +def test_core_is_an_extension_attached_first(): + from tensorrt_llm._torch.attention.backends.core import AttentionCoreMetadata + + metadata = _metadata(_Probe) + extensions = metadata.extensions + assert isinstance(extensions[0], AttentionCoreMetadata) + assert isinstance(extensions[0], MetadataExtension) + assert metadata.core() is extensions[0] + assert type(metadata.core()) is metadata.core_metadata_type + + with pytest.raises(ValueError, match="core"): + metadata.remove_ext(AttentionCoreMetadata) + with pytest.raises(ValueError, match="before"): + metadata._extensions.attach_core(metadata.core_metadata_type(metadata)) + + +def test_extension_cannot_attach_before_the_core(): + from tensorrt_llm._torch.attention.backends.extensions import ExtensionRegistry + + registry = ExtensionRegistry() + with pytest.raises(ValueError, match="core"): + registry.attach(object.__new__(AttentionMetadata), (_Probe,)) + + +def test_core_state_is_forwarded_and_routed_without_init(): + metadata = _metadata() + assert metadata.num_contexts == 0 and metadata.beam_width == 1 + metadata.core().beam_width = 4 + assert metadata.beam_width == 4 + assert "beam_width" not in vars(metadata) + + # Metadata built without __init__ attaches its core on first use. + bare = object.__new__(AttentionMetadata) + bare.core().request_ids = [3] + assert bare.request_ids == [3] + assert bare.extensions[0] is bare.core() + + +def test_constructor_routes_core_arguments_and_rejects_unknown_ones(): + metadata = AttentionMetadata( + max_num_requests=2, max_num_tokens=4, request_ids=[1], beam_width=2 + ) + assert metadata.core().request_ids == [1] + assert metadata.beam_width == 2 + with pytest.raises(TypeError, match="no_such_field"): + AttentionMetadata(max_num_requests=2, max_num_tokens=4, no_such_field=1) + + +def test_copy_owns_its_core_state(): + metadata = _metadata() + metadata.core().request_ids = [1] + clone = copy.copy(metadata) + clone.core().request_ids = [2] + assert metadata.request_ids == [1] + assert clone.core() is not metadata.core() + + +def test_constructor_routes_each_argument_to_its_owner(): + from tensorrt_llm.mapping import Mapping + + mapping = Mapping() + metadata = AttentionMetadata( + max_num_requests=2, max_num_tokens=4, mapping=mapping, draft_kv_cache_manager=None + ) + from tensorrt_llm._torch.attention.backends.features import DraftMetadata + + assert metadata.core().mapping is mapping + assert metadata.ext(DraftMetadata).draft_kv_cache_manager is None + # SpecDecodingMetadata takes max_num_heads_per_kv but this family does not attach it. + with pytest.raises(TypeError, match="SpecDecodingMetadata"): + AttentionMetadata(max_num_requests=2, max_num_tokens=4, max_num_heads_per_kv=4) + # None means not given, even for an owner that is not attached. + AttentionMetadata(max_num_requests=2, max_num_tokens=4, max_num_heads_per_kv=None) + with pytest.raises(TypeError, match="max_num_tokens"): + AttentionMetadata(max_num_requests=2) + + +def test_cross_attention_state_attaches_only_with_cross_attention(): + from tensorrt_llm._torch.attention.backends.features import CrossAttentionMetadata + + metadata = _metadata() + assert metadata.find_ext(CrossAttentionMetadata) is None + assert not metadata.has_cross_sub_metadata + assert not hasattr(metadata, "cross") + + +class _RefreshProbe(MetadataExtension): + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.refreshes = [] + + def refresh_from_draft_lengths(self) -> None: + super().refresh_from_draft_lengths() + self.refreshes.append("refresh_from_draft_lengths") + + +def test_draft_refresh_calls_every_extension_directly(): + metadata = _metadata(_RefreshProbe) + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor([1, 1], dtype=torch.int32), + num_contexts=0, + request_ids=[0, 1], + prompt_lens=[4, 4], + ) + ) + refreshes = metadata.ext(_RefreshProbe).refreshes + refresh_from_lengths(metadata) + assert refreshes == ["refresh_from_draft_lengths"] + + +def test_refresh_draft_runs_only_in_a_draft_loop_and_only_after_changes(): + target = AttentionMetadata( + max_num_requests=2, max_num_tokens=4, one_model_draft=True, extension_types=(_RefreshProbe,) + ) + target.prepare(_draft_batch([1, 1], 0)) + draft_refreshes = target.draft.ext(_RefreshProbe).refreshes + with pytest.raises(RuntimeError, match="draft_loop"): + target.refresh_draft() + with target.draft_loop() as draft: + assert draft is target.draft and target.in_draft_loop + assert not draft.lengths_changed + # Nothing changed: no refresh. + target.refresh_draft() + assert draft_refreshes == [] + draft.set_seq_lens(slice(0, 2), host=2, device=2, update_counts=True) + assert draft.lengths_changed and not target.lengths_changed + with pytest.raises(RuntimeError, match="changed in place"): + draft.check_lengths_current() + target.refresh_draft() + assert draft_refreshes == ["refresh_from_draft_lengths"] + assert not draft.lengths_changed + draft.check_lengths_current() + assert not target.in_draft_loop + # The target's lengths never changed. + assert target.seq_lens.tolist() == [1, 1] + + +def test_lengths_changed_follows_the_length_primitives(): + metadata = _metadata() + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor([2, 2], dtype=torch.int32), + num_contexts=0, + request_ids=[0, 1], + prompt_lens=[4, 4], + ) + ) + assert not metadata.lengths_changed + metadata.check_lengths_current() + metadata.set_seq_lens(slice(0, 2), host=1, device=1, update_counts=True) + assert metadata.lengths_changed + with pytest.raises(RuntimeError, match="changed in place"): + metadata.check_lengths_current() + refresh_from_lengths(metadata) + assert not metadata.lengths_changed + metadata.as_generation_rows() + assert metadata.lengths_changed + metadata.clear_lengths_changed() + assert not metadata.lengths_changed + buffers = metadata.lengths_for_kernel_update() + assert buffers["seq_lens_cuda"] is metadata.seq_lens_cuda + assert metadata.lengths_changed + + +def test_cross_sub_metadata_takes_lengths_from_its_own_inputs(): + decoder = _metadata() + decoder.prepare( + BatchInputs( + seq_lens=torch.tensor([3, 1], dtype=torch.int32), + num_contexts=1, + request_ids=[0, 1], + prompt_lens=[3, 5], + ) + ) + cross = decoder.create_cross_metadata([7, 0]) + assert cross.is_cross + assert cross.seq_lens is decoder.seq_lens + assert cross.seq_lens_cuda is decoder.seq_lens_cuda + assert cross.seq_lens_kv.tolist() == [7, 0] + assert cross.request_ids == [0, 1] and cross.num_contexts == 1 + + # A new decoder batch reaches the sub-metadata only through its own inputs. + decoder.prepare( + BatchInputs( + seq_lens=torch.tensor([1, 1], dtype=torch.int32), + num_contexts=0, + request_ids=[0, 1], + prompt_lens=[3, 5], + ) + ) + assert cross.seq_lens.tolist() == [3, 1] + decoder.use_cross_sub_metadata(cross) + assert decoder.refresh_cross_query_side() is cross + assert cross.seq_lens is decoder.seq_lens and cross.num_contexts == 0 + assert cross.seq_lens_kv.tolist() == [7, 0] + + +class _Workspace(MetadataExtension): + fields = frozenset({"workspace_probe", "scratch_probe"}) + shared = frozenset({"workspace_probe"}) + + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + self.workspace_probe = self.shared_or("workspace_probe", object) + self.scratch_probe = object() + + +def test_copies_are_constructed_and_take_shared_state_from_their_source(): + source = _metadata(_Workspace) + source.core().request_ids = [7] + copy_md = source.constructed_copy(max_num_tokens=8) + assert type(copy_md) is type(source) + assert copy_md.max_num_tokens == 8 and copy_md.request_ids == [7] + assert copy_md.core() is not source.core() + # Declared shared state is the source's; everything else is the copy's own. + assert copy_md.workspace_probe is source.workspace_probe + assert copy_md.scratch_probe is not source.scratch_probe + + +def _draft_batch(seq_lens, num_contexts) -> BatchInputs: + return BatchInputs( + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + num_contexts=num_contexts, + request_ids=list(range(len(seq_lens))), + prompt_lens=[5] * len(seq_lens), + ) + + +def test_draft_metadata_is_built_from_the_target_and_prepared_with_it(): + from tensorrt_llm._torch.attention.backends.features import DraftMetadata + + target = AttentionMetadata( + max_num_requests=2, max_num_tokens=4, one_model_draft=True, extension_types=(_Probe,) + ) + # Built at creation, after every other extension: the draft copies all of them. + assert type(target.extensions[-1]) is DraftMetadata + draft = target.draft + assert draft.find_ext(_Probe) is not None + target.prepare(_draft_batch([2, 1], 1)) + assert draft is target.draft and draft.seq_lens.tolist() == [2, 1] + assert draft.core() is not target.core() + # The draft has no draft of its own, and adopts nothing over a fresh family. + assert not draft.ext(DraftMetadata).one_model_draft + assert draft.find("draft_or_none", None) is None + assert draft.max_num_requests == 2 and draft.kv_cache_manager is target.kv_cache_manager + + # The target's prepare prepares the draft from the same batch. + target.prepare(_draft_batch([1, 1], 0)) + assert draft.seq_lens.tolist() == [1, 1] and draft.num_contexts == 0 + assert draft.seq_lens_cuda is not target.seq_lens_cuda + + # A draft loop changes only the draft's lengths and starts from the target's. + with target.draft_loop() as looped: + assert looped is draft + draft.set_seq_lens(slice(0, 2), device=3) + assert target.seq_lens_cuda.tolist() == [1, 1] + target.refresh_draft() + with target.draft_loop(): + assert draft.seq_lens_cuda.tolist() == [1, 1] + + +def test_metadata_without_one_model_draft_has_no_draft_metadata(): + target = AttentionMetadata(max_num_requests=1, max_num_tokens=1) + with pytest.raises(RuntimeError, match="no one-model draft"): + target.draft + + +def test_cuda_graph_copies_build_their_drafts_over_one_pool(): + target = AttentionMetadata(max_num_requests=2, max_num_tokens=4, one_model_draft=True) + target.prepare(_draft_batch([1, 1], 0)) + graphs = [target.create_cuda_graph_metadata(size, buffers=Buffers()) for size in (1, 2)] + drafts = [graph.draft for graph in graphs] + assert all(draft.is_cuda_graph for draft in drafts) + pools = {id(draft.cuda_graph_buffers) for draft in drafts} + assert len(pools) == 1 + assert drafts[0].cuda_graph_buffers is not graphs[0].cuda_graph_buffers + + +def _all_extension_classes(): + import tensorrt_llm._torch.attention.backends.sparse # noqa: F401 + import tensorrt_llm._torch.attention.backends.trtllm # noqa: F401 + + pending, found = [MetadataExtension], [] + while pending: + cls = pending.pop() + found.append(cls) + pending.extend(cls.__subclasses__()) + return found + + +def test_draft_lists_of_an_adopting_extension_do_not_overlap(): + overlapping = {} + for cls in _all_extension_classes(): + if not cls.adopts_target_state: + continue + bound = cls._declared_union("draft_bound") + written = cls._declared_union("draft_written") + private = cls._declared_union("draft_private") + common = (bound & written) | (bound & private) | (written & private) + if common: + overlapping[cls.__name__] = sorted(common) + assert overlapping == {} + + +def _production_extension_classes(): + import tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step # noqa: F401, E501 + import tensorrt_llm._torch.attention.backends.flashinfer # noqa: F401 + import tensorrt_llm._torch.attention.backends.fp4_mla.state # noqa: F401 + import tensorrt_llm._torch.attention.backends.sparse.csa2.metadata # noqa: F401 + import tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata # noqa: F401 + import tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend # noqa: F401 + import tensorrt_llm._torch.attention.backends.sparse.minimax_m3.triton_metadata # noqa: F401 + import tensorrt_llm._torch.attention.backends.sparse.rocket.metadata # noqa: F401 + import tensorrt_llm._torch.attention.backends.vanilla # noqa: F401 + import tensorrt_llm._torch.models.modeling_qwen4_exp # noqa: F401 + + abstract = {MetadataExtension, MTPIndexShareUser, SparseAlgorithmExtension} + return { + cls.__name__: cls + for cls in _all_extension_classes() + if cls not in abstract and cls.__module__.startswith("tensorrt_llm.") + } + + +def test_resolve_extension_order_deduplicates_subclass_hierarchy(): + from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step import ( # noqa: E501 + GlmTp8DSAMetadata, + ) + from tensorrt_llm._torch.attention.backends.extensions import resolve_extension_order + + order = resolve_extension_order((DSAMetadata, HelixMetadata, GlmTp8DSAMetadata)) + assert DSAMetadata not in order + assert GlmTp8DSAMetadata in order + assert order.index(GlmTp8DSAMetadata) < order.index(HelixMetadata) + + class _DerivedProbe(_Probe): + pass + + metadata = _metadata(_Probe, _DerivedProbe) + assert type(metadata.ext(_Probe)) is _DerivedProbe + assert metadata.ext(_DerivedProbe) is metadata.ext(_Probe) + + class _SiblingA(HelixMetadata): + pass + + class _SiblingB(HelixMetadata): + pass + + with pytest.raises(TypeError, match="Ambiguous subclasses"): + resolve_extension_order((HelixMetadata, _SiblingA, _SiblingB)) + + +def test_every_extension_has_a_creation_config_check(): + """Each production extension names the test checking its constructed copies.""" + from utils.metadata_contract import CREATION_CONFIG_COVERAGE + + production = _production_extension_classes() + assert set(production) == set(CREATION_CONFIG_COVERAGE) + without_arguments = {name for name, test in CREATION_CONFIG_COVERAGE.items() if test is None} + assert all(production[name].declared_init_fields() == () for name in without_arguments) + + +# The hooks every extension implements: construction and the step's prepare. +_LIFECYCLE_HOOKS = frozenset({"__init__", "prepare"}) +# Members an extension may override: how it reports a creation argument, and +# the two draft members only DraftMetadata calls (adoption at the start of a +# draft loop, the refresh after the loop changed the draft's lengths). +_OPT_IN = frozenset({"creation_value", "adopt_target_state", "refresh_from_draft_lengths"}) +# Base methods no extension overrides: shared state and copy bookkeeping. +_BOOKKEEPING = frozenset( + { + "shared_or", + "_plan_draft_copies", + "creation_config", + } +) + + +def test_extensions_override_only_the_lifecycle_hooks(): + base_methods = { + name for name, value in vars(MetadataExtension).items() if inspect.isfunction(value) + } + assert base_methods - _BOOKKEEPING == _LIFECYCLE_HOOKS | _OPT_IN + forbidden = base_methods - _LIFECYCLE_HOOKS - _OPT_IN + overriding = {} + for name, cls in _production_extension_classes().items(): + overridden = forbidden & set(vars(cls)) + if overridden: + overriding[name] = sorted(overridden) + assert overriding == {} + + +def test_draft_member_overrides_call_their_base(): + for name, cls in _production_extension_classes().items(): + for member in ("adopt_target_state", "refresh_from_draft_lengths"): + override = vars(cls).get(member) + if override is not None: + assert f"super().{member}(" in inspect.getsource(override), (name, member) + + +# Ops and kernel launchers that write an attention metadata's length buffers, +# kept next to the ops: each of their arguments that is a length buffer must +# come from the core's lengths_for_kernel_update(), which marks the lengths +# changed so the next forward sees them refreshed. +_LENGTH_WRITING_OPS = frozenset( + { + "step0_epilogue", # speculative/fused_accept.py (Eagle3 / MTP-Eagle step 0) + "k3_spec_accept", # Kimi K3 fused acceptance (trtllm::k3_spec_accept) + "stage_steady_inputs", # SteadyMtpPrepare's staging kernel + "sample_step", # glm sm107 small-batch draft step + "draft_ehproj_table", # glm sm107 draft step on the decode kernels + "draft_sample_glue", + } +) +_LENGTH_BUFFERS = frozenset({"kv_lens_cuda", "seq_lens_cuda"}) +# Where the engine, models, modeling_v2 targets and speculative workers live. +_LENGTH_WRITER_ROOTS = ("pyexecutor", "models", "_experimental/modeling_v2", "speculative") + + +def _reads_length_buffer_directly(node) -> bool: + """Whether ``node`` is ``.kv_lens_cuda`` / ``.seq_lens_cuda``, sliced or not.""" + import ast + + while isinstance(node, ast.Subscript): + node = node.value + return isinstance(node, ast.Attribute) and node.attr in _LENGTH_BUFFERS + + +def test_kernels_write_length_buffers_only_from_lengths_for_kernel_update(): + import ast + import pathlib + + import tensorrt_llm._torch as torch_backend + + root = pathlib.Path(torch_backend.__file__).parent + offenders = [] + for base in _LENGTH_WRITER_ROOTS: + for path in (root / base).rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) + if name not in _LENGTH_WRITING_OPS: + continue + arguments = [*node.args, *(keyword.value for keyword in node.keywords)] + for argument in arguments: + candidates = [argument] + if isinstance(argument, ast.IfExp): + candidates = [argument.body, argument.orelse] + if any(_reads_length_buffer_directly(c) for c in candidates): + offenders.append(f"{path.relative_to(root)}:{node.lineno} {name}") + assert offenders == [] + + +@pytest.mark.parametrize("metadata_type", [TrtllmAttentionMetadata, VanillaAttentionMetadata]) +def test_context_rows_continue_as_generation_rows(metadata_type): + metadata = object.__new__(metadata_type) + core = metadata.core() + lengths = torch.tensor([5, 3, 4, 4], dtype=torch.int32) + core.bind_lengths(lengths, lengths.clone(), num_contexts=2) + if metadata_type is TrtllmAttentionMetadata: + # The per-row request types (0: context, 1: generation) and the + # cached-context-KV state, as a prepare of this batch leaves them. + core.host_request_types = torch.tensor([0, 0, 1, 1], dtype=torch.int32) + core.max_ctx_kv_len = 9 + core.num_ctx_cached_tokens = 4 + assert (metadata.num_contexts, metadata.num_ctx_tokens) == (2, 8) + + # The draft loop's step-0 update: one token per row from here on. + metadata.set_seq_lens(slice(0, 4), host=1, device=1) + metadata.as_generation_rows() + + assert metadata.num_contexts == 0 + assert metadata.num_generations == 4 + assert metadata.num_ctx_tokens == 0 + assert metadata.num_tokens == 4 + if metadata_type is TrtllmAttentionMetadata: + assert core.host_request_types.tolist() == [1, 1, 1, 1] + assert core.max_ctx_kv_len == 0 + assert core.num_ctx_cached_tokens == 0 + + +@needs_cuda +def test_prepare_applies_kv_lens_offsets_to_the_device_lengths_only(): + def batch(**changes): + # Attention without a KV cache: the KV lengths are the query lengths. + inputs = dict( + seq_lens=torch.tensor([6, 9], dtype=torch.int32), + num_contexts=2, + request_ids=[0, 1], + prompt_lens=[6, 9], + kv_cache_params=KVCacheParams(use_cache=False), + ) + return BatchInputs(**{**inputs, **changes}) + + metadata = TrtllmAttentionMetadata( + max_num_requests=2, max_num_tokens=8, max_seq_len=32, one_model_draft=True + ) + metadata.prepare(batch()) + buffer = metadata.kv_lens_cuda + + offsets = torch.tensor([-1, 2], dtype=torch.int32, device="cuda") + metadata.prepare(batch(kv_lens_offsets=offsets)) + assert metadata.kv_lens_cuda is buffer + assert metadata.kv_lens_cuda[:2].tolist() == [5, 11] + # The host lengths stay the scheduled ones; the device knows the correction. + assert metadata.kv_lens[:2].tolist() == [6, 9] + # The draft takes the corrected lengths when its loop starts. + start_draft_loop(metadata) + assert metadata.draft.kv_lens_cuda[:2].tolist() == [5, 11] + + # A step without corrections derives the lengths from the host alone. + metadata.prepare(batch()) + assert metadata.kv_lens_cuda[:2].tolist() == [6, 9] + + # A metadata without device KV lengths rejects the correction. + with pytest.raises(ValueError, match="kv_lens_offsets"): + _metadata().prepare(batch(kv_lens_offsets=offsets)) diff --git a/tests/unittest/_torch/attention/hw_agnostic/test_ragged_metadata_lifecycle.py b/tests/unittest/_torch/attention/hw_agnostic/test_ragged_metadata_lifecycle.py index 07d1eb1befe6..5467b2e18780 100644 --- a/tests/unittest/_torch/attention/hw_agnostic/test_ragged_metadata_lifecycle.py +++ b/tests/unittest/_torch/attention/hw_agnostic/test_ragged_metadata_lifecycle.py @@ -3,35 +3,45 @@ """CPU coverage of ragged preparation, update and backend dispatch contracts.""" from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest import torch +from _torch.helpers import bare_extension, refresh_from_lengths +from tensorrt_llm._torch.attention.backends.features import DraftMetadata, FlashMlaMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.interface import AttentionInputType from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.module import ( forward_generation_sparse_attn, ) from tensorrt_llm._torch.attention.backends.sparse.dsa.indexer import Indexer -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata pytestmark = pytest.mark.cpu_only -def _metadata(seq_lens: list[int], num_contexts: int = 0) -> DSAtrtllmAttentionMetadata: - metadata = DSAtrtllmAttentionMetadata.__new__(DSAtrtllmAttentionMetadata) +def _metadata(seq_lens: list[int], num_contexts: int = 0) -> TrtllmAttentionMetadata: + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) metadata._seq_lens = torch.tensor(seq_lens, dtype=torch.int32) - metadata._seq_lens_cuda = metadata._seq_lens.clone() + metadata._seq_lens_cuda = metadata.seq_lens.clone() metadata._num_contexts = num_contexts metadata._num_generations = len(seq_lens) - num_contexts metadata._num_ctx_tokens = sum(seq_lens[:num_contexts]) metadata._num_tokens = sum(seq_lens) - metadata.max_draft_tokens = 5 - metadata.runtime_tokens_per_gen_step = 0 - metadata._indexer_compress_ratio = 1 metadata.kv_cache_manager = None - metadata.num_sms = 1 + with patch( + "tensorrt_llm.deep_gemm.get_num_sms", + return_value=1, + ): + dsa = bare_extension(DSAMetadata, metadata) + dsa.max_draft_tokens = 5 + dsa._indexer_compress_ratio = 1 + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + metadata.replace_ext(dsa) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) return metadata @@ -48,7 +58,6 @@ def test_schedule_update_keeps_runtime_query_width(monkeypatch, runtime_width: i metadata.gen_kv_indptr = torch.zeros(3, dtype=torch.int64) metadata.gen_cached_token_indptr = torch.zeros(3, dtype=torch.int64) metadata.enable_ragged_verification = False - metadata.enable_flash_mla = False metadata.use_fp8_ds_mla = False metadata.use_expanded_buffers_for_mtp = False metadata.expand_for_dsl = False @@ -72,7 +81,7 @@ def schedule(context_lens, block_kv, num_sms): ) Indexer.prepare_scheduler_metadata(metadata) metadata.kv_lens_cuda.add_(1) - metadata.on_update_kv_lens() + refresh_from_lengths(metadata) assert [tuple(value.shape) for value in scheduler_inputs] == [ (2, 1), @@ -92,8 +101,8 @@ def test_prepare_initializes_row_extents_and_refresh_reuses_storage( ) -> None: metadata = _metadata(([3] if num_contexts else []) + [2, 4], num_contexts) metadata.ragged_verify_lens = [2, 4] + dsa = metadata.ext(DSAMetadata) metadata.enable_ragged_verification = True - metadata.is_spec_decoding_enabled = False metadata.sparse_metadata_params = SimpleNamespace(use_cute_dsl_paged_mqa_logits=False) metadata.gen_token_repeats_cuda = torch.empty(2, dtype=torch.int64) metadata.host_gen_token_repeats = torch.empty(2, dtype=torch.int64) @@ -102,11 +111,11 @@ def test_prepare_initializes_row_extents_and_refresh_reuses_storage( for name in ("row_kv_lens", "row_kv_correction", "row_req_idx"): dtype = torch.long if name == "row_req_idx" else torch.int32 for suffix in ("host", "cuda"): - setattr(metadata, f"{name}_{suffix}", torch.full((6,), -777, dtype=dtype)) + setattr(dsa, f"{name}_{suffix}", torch.full((6,), -777, dtype=dtype)) for name in ("attn_row_kv_lens", "attn_row_kv_correction", "attn_row_req_idx"): dtype = torch.long if name == "attn_row_req_idx" else torch.int32 for suffix in ("host", "cuda"): - setattr(metadata, f"{name}_{suffix}", torch.full((7,), -777, dtype=dtype)) + setattr(dsa, f"{name}_{suffix}", torch.full((7,), -777, dtype=dtype)) metadata.prompt_lens_cpu = torch.tensor(([3] if num_contexts else []) + [100, 200]) metadata.attn_row_prompt_lens_cpu = torch.empty(7, dtype=torch.int32) metadata.attn_row_prompt_lens_cuda = torch.empty(7, dtype=torch.int32) @@ -152,7 +161,7 @@ def _prefix_metadata(monkeypatch, num_contexts=0, ragged=True): metadata.ragged_verify_lens = [1, 2, 4] if ragged else None metadata.max_num_sequences = len(seq_lens) metadata.cuda_graph_buffers = {} - metadata.draft_kv_cache_manager = None + metadata.replace_ext(bare_extension(DraftMetadata, metadata)) metadata.kv_cache_manager = SimpleNamespace( num_pools=1, num_attention_op_pools=1, max_blocks_per_seq=1 ) @@ -176,11 +185,10 @@ def allocate(buffers, shape, *, cache_name, dtype, capture_graph): metadata.mla_cu_q_rows = torch.zeros(len(seq_lens) + 1, dtype=torch.int32) metadata.mla_cu_kv_seqlens = torch.zeros(len(seq_lens) + 1, dtype=torch.int32) metadata.mla_ctx_cu_q_seqlens = torch.zeros(len(seq_lens) + 1, dtype=torch.int32) - metadata._mla_ctx_cu_seqlens_valid = False - metadata._mla_scheduler_buffers_valid = False + metadata.core()._mla_ctx_cu_seqlens_valid = False + metadata.core()._mla_scheduler_buffers_valid = False metadata.prompt_lens_cpu = metadata.seq_lens.clone() metadata.kv_cache_block_offsets = None - metadata.is_spec_decoding_enabled = False metadata.sparse_metadata_params = SimpleNamespace(use_cute_dsl_paged_mqa_logits=False) monkeypatch.setattr( "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata.get_sm_version", @@ -209,8 +217,8 @@ def test_feature_off_keeps_request_major_prefixes_and_allocations(monkeypatch): def test_uniform_step_with_opt_in_keeps_request_major_prefixes(monkeypatch): metadata, _ = _prefix_metadata(monkeypatch) metadata.ragged_verify_lens = None - metadata._seq_lens.fill_(2) - metadata._seq_lens_cuda.copy_(metadata._seq_lens) + metadata.seq_lens.fill_(2) + metadata.seq_lens_cuda.copy_(metadata.seq_lens) q_prefix, kv_prefix = metadata.mla_prepare_scheduler_buffers(2) assert q_prefix.data_ptr() == metadata.mla_cu_q_rows.data_ptr() torch.testing.assert_close(q_prefix, torch.tensor([0, 4, 8, 12], dtype=torch.int32)) @@ -275,10 +283,10 @@ def launch(): launch() assert observed[-1][2:] == observed[-2][2:] - # Exercise the actual overlap hook: refresh device rows and invalidate the - # cached prefix before the next module-level dispatch, without host refresh. + # Exercise the device refresh after a draft step: refresh device rows and + # invalidate the cached prefix before the next module-level dispatch, + # without host refresh. metadata.kv_cache_manager = None - metadata.enable_flash_mla = False metadata.use_fp8_ds_mla = False metadata.expand_for_dsl = False metadata.kv_lens_cuda_2d = torch.empty(metadata.num_generations, 6, dtype=torch.int32) @@ -295,8 +303,8 @@ def launch(): lambda *unused: torch.zeros(1, dtype=torch.int32), ) metadata.kv_lens_cuda[num_contexts:] -= torch.tensor([2, 3, 6]) - metadata.on_update_kv_lens() - assert metadata._mla_scheduler_buffers_valid is False + refresh_from_lengths(metadata) + assert metadata._ragged_mla_buffers_valid is False launch() expected_kv -= torch.tensor([2, 3, 3, 6, 6, 6, 6]) torch.testing.assert_close(observed[-1][0], expected_q) @@ -308,9 +316,10 @@ def launch(): @pytest.mark.parametrize("metadata_valid", [False, True]) def test_flash_mla_rejects_token_major_generation_before_dispatch(metadata_valid: bool) -> None: metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) - metadata.enable_flash_mla = True - metadata._flash_mla_metadata_valid = metadata_valid - metadata.token_major_gen_view = Mock(return_value=object()) + flash_mla = bare_extension(FlashMlaMetadata, metadata) + flash_mla.metadata_valid = metadata_valid + metadata.replace_ext(flash_mla) + metadata.core().token_major_view = Mock(return_value=object()) attention = SimpleNamespace( sparse_params=None, is_mla_enable=False, @@ -336,8 +345,9 @@ def test_uniform_flash_mla_and_ragged_native_pass_the_guard( flash_mla: bool, token_major: bool ) -> None: metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) - metadata.enable_flash_mla = flash_mla - metadata.token_major_gen_view = Mock(return_value=object() if token_major else None) + if flash_mla: + metadata.replace_ext(bare_extension(FlashMlaMetadata, metadata)) + metadata.core().token_major_view = Mock(return_value=object() if token_major else None) metadata.use_paged_context_fmha = False attention = SimpleNamespace( sparse_params=None, diff --git a/tests/unittest/_torch/attention/multi_gpu/helix_test_utils.py b/tests/unittest/_torch/attention/multi_gpu/helix_test_utils.py index b7afc60e2113..90c2033f6e47 100644 --- a/tests/unittest/_torch/attention/multi_gpu/helix_test_utils.py +++ b/tests/unittest/_torch/attention/multi_gpu/helix_test_utils.py @@ -20,6 +20,7 @@ from mpi4py.futures import MPIPoolExecutor import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.interface import KVCacheParams from tensorrt_llm._torch.attention.backends.utils import get_attention_backend from tensorrt_llm._torch.nccl_window_tensor_scope import discard_nccl_window_tensor_outputs @@ -82,6 +83,7 @@ def split_inputs_for_rank(input_ctx, position_ids_ctx, scenario, rank, world_siz def activate_all_ranks_for_context(attn_metadata, position_ids): """Override helix metadata so all ranks participate in the context step.""" + attn_metadata.add_ext(HelixMetadata) attn_metadata.helix_position_offsets = position_ids if ( hasattr(attn_metadata, "helix_is_inactive_rank") @@ -126,7 +128,7 @@ def create_helix_gen_metadata( ), **kwargs, ) - attn_metadata.enable_helix = True + attn_metadata.add_ext(HelixMetadata) attn_metadata.helix_is_inactive_rank = torch.tensor( helix_is_inactive_rank, dtype=torch.bool, device="cuda" ) diff --git a/tests/unittest/_torch/attention/multi_gpu/test_mla_helix_context.py b/tests/unittest/_torch/attention/multi_gpu/test_mla_helix_context.py index 486604ae8f35..060135c2634c 100644 --- a/tests/unittest/_torch/attention/multi_gpu/test_mla_helix_context.py +++ b/tests/unittest/_torch/attention/multi_gpu/test_mla_helix_context.py @@ -28,6 +28,7 @@ from mpi4py import MPI from utils.util import skip_pre_blackwell +from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures, KVCacheParams from tensorrt_llm._torch.attention.backends.utils import get_attention_backend from tensorrt_llm._torch.configs.kimi_linear import KimiLinearConfig @@ -236,6 +237,9 @@ def _check_context_heads(rank, world_size, scenario, use_nccl_for_alltoall, fifo # The generation kernel emits CP softmax statistics only when its # Helix fields are present. Every rank appends to its full replica in # this item 2 fixture; ownership is tested separately in item 3. + # A CP mapping attaches Helix state by itself; the unsharded baseline + # asks for it. + metadata.add_ext(HelixMetadata) metadata.update_helix_param([context_length], [False]) metadata.prepare() extra_attrs["attention_metadata"] = weakref.ref(metadata) @@ -364,6 +368,7 @@ def metadata_for( query_lengths[i] if i < num_contexts else lengths[i] for i in range(batch) ] metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached) + metadata.add_ext(HelixMetadata) if global_positions is not None: metadata.update_helix_param( global_positions, diff --git a/tests/unittest/_torch/attention/sparse/csa2/_utils.py b/tests/unittest/_torch/attention/sparse/csa2/_utils.py index f105e24ee496..8f907c3e3e2d 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/_utils.py +++ b/tests/unittest/_torch/attention/sparse/csa2/_utils.py @@ -2,13 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 """Independent CSA2 test fixtures and mathematical references.""" +import dataclasses from copy import copy from types import SimpleNamespace import torch +from _torch.helpers import bare_extension +from tensorrt_llm._torch.attention.backends.core import AttentionCoreMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.indexer import CSA2Indexer -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import ( CSA2ForwardState, CSA2Layout, @@ -24,6 +28,15 @@ store_index_rows, store_rows, ) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + + +def _bare_csa2_metadata() -> TrtllmAttentionMetadata: + """Metadata built without ``__init__``, carrying a bare CSA2 extension.""" + metadata = object.__new__(TrtllmAttentionMetadata) + metadata.replace_ext(bare_extension(CSA2Metadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) + return metadata def _index_pages(rows: int, device="cpu") -> torch.Tensor: @@ -80,7 +93,7 @@ def _attention_reference(q, kv, valid, sink): def _page_metadata(table, requests, page_size, max_positions): # Exercise device-independent metadata transforms without allocating native # compute buffers. Constructor/lifecycle coverage uses real V2 managers. - meta = object.__new__(CSA2TrtllmMetadata) + meta = _bare_csa2_metadata() meta.csa2_global_page_tables = {0: table} meta.csa2_token_requests = requests meta.csa2_global_page_sizes = {0: page_size} @@ -130,7 +143,10 @@ def _run_indexer(layout, layer_idx, state): layer = layout.layer(layer_idx) meta.enter_layer(layer) manager.write_swa( - layer_idx, meta.csa2_swa_write_slots[layer_idx], state.swa_kv, state.swa_transform + layer_idx, + meta.csa2_swa_write_slots[layer_idx], + state.swa_kv, + state.swa_transform, ) if layer.mode == CSA2Mode.FULL: manager.write_global( @@ -265,6 +281,9 @@ def _run_modes(device, graph_replay=False): def run(): meta = copy(base) + routing = copy(base.ext(CSA2Metadata)) + routing.metadata = meta + meta.replace_ext(routing) meta.reset_routing() outputs = [] for i in range(3): @@ -324,24 +343,56 @@ def __init__(self, enable_attention_dp: bool): self.enable_attention_dp = enable_attention_dp +@dataclasses.dataclass class _FakeKvParams: - def __init__(self, num_cached): - self.num_cached_tokens_per_seq = list(num_cached) + num_cached_tokens_per_seq: list class _FakeMetadata: """The attributes ``plan_decoder_replay`` reads, plus the SWA floors and slot resolution. - A real ``CSA2TrtllmMetadata`` needs a KV cache manager, pools + A real CSA2 metadata needs a KV cache manager, pools and a device; the planner is pure host arithmetic over these fields, so standing them up here keeps this a CPU test and keeps the refusals -- which are the part worth pinning -- independent of cache-manager plumbing. ``prepare()`` is modelled too, because the boundary's correctness depends on - what it clears; see the method. + what it clears; see the method. Its decoder replay metadata is a copy of + it with tensors and publications of its own. """ - reset_routing = CSA2TrtllmMetadata.reset_routing + reset_routing = CSA2Metadata.reset_routing + prepare_decoder_replay = CSA2Metadata.prepare_decoder_replay + set_all_rank_num_tokens = AttentionCoreMetadata.set_all_rank_num_tokens + decoder_context_ends = CSA2Metadata.decoder_context_ends + csa2_remote_tail_mode = CSA2Metadata.csa2_remote_tail_mode + csa2_remote_tail_starts = CSA2Metadata.csa2_remote_tail_starts + + @property + def metadata(self) -> "_FakeMetadata": + # The fake serves as both the core metadata and its CSA2 extension. + return self + + def ext(self, cls: type) -> "_FakeMetadata": + return self + + find_ext = ext + + def core(self) -> "_FakeMetadata": + return self + + @property + def decoder_replay_metadata(self) -> "_FakeMetadata": + if self._fake_replay is None: + replay = copy(self) + for name, value in vars(self).items(): + if isinstance(value, torch.Tensor): + setattr(replay, name, value.clone()) + elif isinstance(value, (dict, set)): + setattr(replay, name, type(value)()) + replay.prepare_calls = 0 + self._fake_replay = replay + return self._fake_replay def __init__( self, @@ -361,7 +412,15 @@ def __init__( self.is_cuda_graph = captured self.mapping = _FakeMapping(attention_dp) self.kv_cache_manager = _FakeManager(swa_reuse) - self.kv_cache_params = _FakeKvParams(num_cached or [0] * len(seq_lens)) + self.kv_cache_params = _FakeKvParams(list(num_cached or [0] * len(seq_lens))) + self.kv_lens_offsets = None + self.num_chunked_ctx_requests = 0 + self.beam_width = 1 + self.request_ids = None + self.all_rank_num_tokens = None + self.csa2_precomputed_kv_layers = set() + self._fake_replay = None + self.csa2_replay_query_rows = None self.csa2_candidates = {} self.csa2_candidate_blocks = {} self.csa2_candidate_counts = {} @@ -378,7 +437,26 @@ def __init__( self.csa2_replay_start_positions = torch.zeros(self.num_seqs, dtype=torch.int64) self._csa2_swa_resolved = False - def prepare(self): + def apply_batch_inputs(self, inputs) -> None: + self.seq_lens = inputs.seq_lens + self.seq_lens_cuda = self.seq_lens + self.num_contexts = inputs.num_contexts + self.num_seqs = len(inputs.seq_lens) + self.num_tokens = int(inputs.seq_lens.sum()) + self.request_ids = inputs.request_ids + self.prompt_lens = inputs.prompt_lens + self.kv_cache_params = inputs.kv_cache_params + self.num_chunked_ctx_requests = inputs.num_chunked_ctx_requests + self.kv_lens_offsets = inputs.kv_lens_offsets + + def live_cached_lengths(self) -> torch.Tensor: + """The cached lengths with the KV-length correction a prepare applies.""" + cached = torch.tensor(self.kv_cache_params.num_cached_tokens_per_seq[: self.num_seqs]) + if self.kv_lens_offsets is not None: + cached = cached + self.kv_lens_offsets[: self.num_seqs].cpu() + return cached + + def prepare(self, inputs=None): """Only the part of ``prepare()`` the boundary has to survive. The real one rebuilds every device index buffer; what matters to @@ -387,6 +465,8 @@ def prepare(self): boundary testable on CPU and keeps the test honest about *why* the reinstatement exists. """ + if inputs is not None: + self.apply_batch_inputs(inputs) self.prepare_calls += 1 self.csa2_indices.clear() self.csa2_candidates.clear() @@ -399,7 +479,7 @@ def prepare(self): [ torch.arange(start, start + length) for start, length in zip( - self.kv_cache_params.num_cached_tokens_per_seq, self.seq_lens.tolist() + self.live_cached_lengths().tolist(), self.seq_lens.tolist() ) ] ) @@ -420,11 +500,3 @@ def _ensure_swa_slots(self): self.csa2_swa_indices = {0: slots.masked_fill(logical < read_floor[:, None], -1)} self.csa2_swa_write_slots = {0: slots[:, -1]} self.csa2_visible_lengths = {0: self.csa2_positions + 1} - - def on_update_kv_lens(self): - self._csa2_swa_resolved = False - lengths = self.seq_lens.tolist() - starts = self.kv_lens_cuda[: self.num_seqs] - self.seq_lens - self.csa2_positions = torch.cat( - [torch.arange(s, s + n) for s, n in zip(starts.tolist(), lengths)] - ) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_backend.py b/tests/unittest/_torch/attention/sparse/csa2/test_backend.py index e8f5fea6d206..f207a5cfe5e9 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_backend.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_backend.py @@ -8,7 +8,12 @@ import torch from utils.util import skip_blackwell_geforce -from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2FlashInfer, CSA2FlashMLA +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import ( + CSA2FlashInfer, + CSA2FlashMLA, + CSA2TrtllmAttention, +) +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Params skip_pre_blackwell = pytest.mark.skipif( @@ -173,9 +178,8 @@ def record(attn, *args, **kwargs): extra_valid[0, 1::2] = False # holes before later valid entries extra_valid[1] = False sink = torch.randn(heads, device="cuda") - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - metadata = CSA2TrtllmMetadata.for_query_tile(q, extra_width) + metadata = CSA2Metadata.for_query_tile(q, extra_width) args = _inputs(q, swa, extra, swa_valid, extra_valid, sink) out = attn.forward(q.flatten(1), None, None, metadata, forward_args=args).view_as(q) torch.cuda.synchronize() @@ -197,10 +201,9 @@ def test_trtllm_graph_replay_resets_sparse_state(): extra_valid = torch.ones(2, 17, device="cuda", dtype=torch.bool) sink = torch.zeros(64, device="cuda") - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import pack_rows - frame = CSA2TrtllmMetadata.for_query_tile(q, 17) + frame = CSA2Metadata.for_query_tile(q, 17) def run(): frame.is_cuda_graph = torch.cuda.is_current_stream_capturing() @@ -217,7 +220,7 @@ def run(): # Larger eager queries use independent caller-owned metadata. No recursive # warmup or backend-private frame allocation is needed. large_q = q.repeat(8, 1, 1) - large_frame = CSA2TrtllmMetadata.for_query_tile(large_q, 17) + large_frame = CSA2Metadata.for_query_tile(large_q, 17) large_args = _inputs( large_q, swa.repeat(8, 1, 1), @@ -265,7 +268,6 @@ def test_standard_backend_multiple_tiles_and_reuse(monkeypatch): CSA2CacheRole, ) from tensorrt_llm._torch.attention.backends.sparse.csa2.indexer import CSA2Indexer - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import ( CSA2BackendForwardArgs, CSA2ForwardState, @@ -303,7 +305,7 @@ def predict(indexer, metadata, hidden_states, *args, **kwargs): request = manager._create_kv_cache(100, None, []) assert manager._resume_and_restore(100, request) assert request.resize(count) - runtime = CSA2TrtllmMetadata( + runtime = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=count, kv_cache_manager=manager ) positions = torch.arange(count, device="cuda") @@ -379,15 +381,13 @@ def predict(indexer, metadata, hidden_states, *args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_shared_metadata_resets_global_to_swa_inputs(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - q = torch.randn(2, 8, 512, device="cuda", dtype=torch.bfloat16) swa = torch.randn(2, 4, 512, device="cuda", dtype=torch.bfloat16) main = torch.randn(2, 17, 512, device="cuda", dtype=torch.bfloat16) swa_valid = torch.ones(2, 4, device="cuda", dtype=torch.bool) main_valid = torch.ones(2, 17, device="cuda", dtype=torch.bool) sink = torch.zeros(8, device="cuda") - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17) + metadata = CSA2Metadata.for_query_tile(q, 17) first, second = _backend(8, 20), _backend(8, 21) args = _inputs(q, swa, main, swa_valid, main_valid, sink) first.forward(q.flatten(1), None, None, metadata, forward_args=args) @@ -408,14 +408,12 @@ def test_shared_metadata_resets_global_to_swa_inputs(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_cold_metadata_rejects_capture(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - attn = _backend(8) q = torch.zeros(2, 8, 512, device="cuda", dtype=torch.bfloat16) swa = torch.zeros(2, 1, 512, device="cuda", dtype=torch.bfloat16) valid = torch.ones(2, 1, device="cuda", dtype=torch.bool) sink = torch.zeros(8, device="cuda") - metadata = CSA2TrtllmMetadata.for_query_tile(q, 0) + metadata = CSA2Metadata.for_query_tile(q, 0) args = _inputs(q, swa, None, valid, None, sink) metadata.is_cuda_graph = True graph = torch.cuda.CUDAGraph() @@ -433,7 +431,6 @@ def test_cold_metadata_rejects_capture(): @torch.inference_mode() def test_alternative_helper_dispatch(implementation, context, monkeypatch): from tensorrt_llm._torch.attention.backends.fmha.fallback import FallbackFmha - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata def unexpected_fallback(*args, **kwargs): raise AssertionError("Selected CSA2 implementation fell through to native fallback") @@ -449,7 +446,7 @@ def unexpected_fallback(*args, **kwargs): swa_valid[1] = False main_valid[1] = False sink = torch.linspace(-2, 2, 8, device="cuda") - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17) + metadata = CSA2Metadata.for_query_tile(q, 17) args = _inputs(q, swa, main, swa_valid, main_valid, sink) if context: from tensorrt_llm._torch.attention.backends.interface import AttentionInputType @@ -472,7 +469,6 @@ def unexpected_fallback(*args, **kwargs): @torch.inference_mode() def test_alternative_backend_rejects_unsupported_options(option): from tensorrt_llm._torch.attention.backends.interface import CustomAttentionMask - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata attn = _backend(8, compute_backend="flashinfer") q = torch.zeros(1, 8, 512, device="cuda", dtype=torch.bfloat16) @@ -490,7 +486,7 @@ def test_alternative_backend_rejects_unsupported_options(option): else: args.output = torch.empty_like(q).flatten(1) args.output_sf = torch.empty(1, device="cuda", dtype=torch.uint8) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 0) + metadata = CSA2Metadata.for_query_tile(q, 0) with pytest.raises(ValueError, match="(do not support.*mask/output format|output must match)"): _helper_forward(attn, q.flatten(1), metadata, args) @@ -499,14 +495,12 @@ def test_alternative_backend_rejects_unsupported_options(option): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_alternative_helper_revalidates_output_scale(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - attn = _backend(8, compute_backend="flashinfer") q = torch.zeros(1, 8, 512, device="cuda", dtype=torch.bfloat16) swa = torch.zeros(1, 1, 512, device="cuda", dtype=torch.bfloat16) valid = torch.ones(1, 1, device="cuda", dtype=torch.bool) args = _inputs(q, swa, None, valid, None, torch.zeros(8, device="cuda")) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 0) + metadata = CSA2Metadata.for_query_tile(q, 0) _helper_forward(attn, q.flatten(1), metadata, args) torch.cuda.synchronize() args.out_scale = torch.ones(1, device="cuda") @@ -523,7 +517,6 @@ def test_alternative_helper_revalidates_output_scale(): def test_native_context_preserves_real_query_groups(extra_width, prefix, tile_start, monkeypatch): from tensorrt_llm._torch.attention.backends.fmha.fallback import FallbackFmha from tensorrt_llm._torch.attention.backends.interface import AttentionInputType - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata torch.manual_seed(459) heads, count = 64, 5 @@ -545,11 +538,14 @@ def test_native_context_preserves_real_query_groups(extra_width, prefix, tile_st extra_valid[1, ::3] = False extra_valid[2] = False sink = torch.randn(heads, device="cuda") - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=count + tile_start) + source = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=count + tile_start) source._num_contexts = 2 source._num_ctx_tokens = source._num_tokens = count + tile_start source.csa2_num_context_requests = 2 - source.csa2_request_query_ranges = ((0, 3 + tile_start), (3 + tile_start, count + tile_start)) + source.csa2_request_query_ranges = ( + (0, 3 + tile_start), + (3 + tile_start, count + tile_start), + ) source.csa2_request_start_positions = (prefix, prefix * 2) source.csa2_request_lengths = (3 + tile_start, 2) metadata = source.get_query_tile_metadata(q, extra_width, query_start=tile_start) @@ -587,7 +583,7 @@ def record(provider, query, key, value, meta, forward_args): ) monkeypatch.setattr(FallbackFmha, "forward", original) - generation = CSA2TrtllmMetadata.for_query_tile(q, extra_width) + generation = CSA2Metadata.for_query_tile(q, extra_width) generation_args = _inputs(q, swa, extra, swa_valid, extra_valid, sink) reference_generation = attn.forward( q.flatten(1), None, None, generation, forward_args=generation_args @@ -604,7 +600,6 @@ def test_packed_helper_skips_bf16_staging(extra_width, monkeypatch): pytest.skip("SM100 packed attention") from tensorrt_llm._torch.attention.backends.fmha.fallback import FallbackFmha - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata torch.manual_seed(460) heads, count = 32, 3 @@ -632,9 +627,11 @@ def test_packed_helper_skips_bf16_staging(extra_width, monkeypatch): records = torch.full((packed.shape[0], 356), 73, dtype=torch.uint8, device="cuda") records[:, :288].copy_(packed) args.sparse_backend_args.main_pool = records[:, :288] - metadata = CSA2TrtllmMetadata.for_query_tile(q, extra_width) + metadata = CSA2Metadata.for_query_tile(q, extra_width) monkeypatch.setattr( - metadata, "stage_selected", lambda *a: pytest.fail("Packed FMHA staged BF16 rows") + metadata.ext(CSA2Metadata), + "stage_selected", + lambda *a: pytest.fail("Packed FMHA staged BF16 rows"), ) output = _helper_forward(attn, q.flatten(1), metadata, args).view_as(q) torch.cuda.synchronize() @@ -691,7 +688,6 @@ def test_native_backend_rejects_disabled_fallback(monkeypatch): @torch.inference_mode() def test_context_graph_uses_fixed_generation_frame(): from tensorrt_llm._torch.attention.backends.interface import AttentionInputType - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata torch.manual_seed(461) heads, count, extra_width = 64, 5, 17 @@ -702,7 +698,7 @@ def test_context_graph_uses_fixed_generation_frame(): swa_valid = torch.ones(count, 128, device="cuda", dtype=torch.bool) extra_valid = torch.ones(count, extra_width, device="cuda", dtype=torch.bool) sink = torch.randn(heads, device="cuda") - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=count) + source = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=count) source._num_contexts = 2 source._num_ctx_tokens = source._num_tokens = count source.csa2_num_context_requests = 2 @@ -759,13 +755,11 @@ def test_context_graph_uses_fixed_generation_frame(): @pytest.mark.parametrize("implementation,packed", [("flashinfer", False), ("trtllm", True)]) @torch.inference_mode() def test_direct_compute_rejects_native_forward_misuse(implementation, packed): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - attn = _backend(32, compute_backend=implementation, use_packed=packed) q = torch.zeros(1, 32, 512, device="cuda", dtype=torch.bfloat16) values = torch.zeros(1, 1, 512, device="cuda", dtype=torch.bfloat16) valid = torch.ones(1, 1, device="cuda", dtype=torch.bool) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 0) + metadata = CSA2Metadata.for_query_tile(q, 0) args = _inputs(q, values, None, valid, None, torch.zeros(32, device="cuda")) with pytest.raises(ValueError, match="explicit module helper"): attn.forward(q.flatten(1), None, None, metadata, forward_args=args) @@ -953,7 +947,6 @@ def test_global_slot_graph_mapping_folds_visibility(monkeypatch): def test_native_fp8_quantized_attention(phase, extra_width, heads, monkeypatch, record_property): from tensorrt_llm._torch.attention.backends.fmha.fallback import FallbackFmha from tensorrt_llm._torch.attention.backends.interface import AttentionInputType - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import unpack_rows torch.manual_seed(482) @@ -984,11 +977,9 @@ def test_native_fp8_quantized_attention(phase, extra_width, heads, monkeypatch, args = _inputs(q, swa, extra, sv, ev, sink) # Staging derives both scales from the packed pools; the oracle below reads # them back off the metadata and the forward args. - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=count) + source = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=count) if phase == "generation": - metadata = CSA2TrtllmMetadata.for_query_tile( - q, extra_width, staging_dtype=attn.staging_dtype - ) + metadata = CSA2Metadata.for_query_tile(q, extra_width, staging_dtype=attn.staging_dtype) else: source._num_contexts = 2 source._num_ctx_tokens = source._num_tokens = count @@ -1152,7 +1143,6 @@ def test_native_fp8_staging_keeps_rows_beyond_the_e4m3_range(phase, extra_width) from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import ( _CONTEXT_STAGE_SCALE_MAX_EXPONENT, ) - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import unpack_rows torch.manual_seed(482) @@ -1170,11 +1160,9 @@ def test_native_fp8_staging_keeps_rows_beyond_the_e4m3_range(phase, extra_width) sink = torch.randn(heads, device="cuda") args = _inputs(q, swa, extra, sv, ev, sink) if phase == "generation": - metadata = CSA2TrtllmMetadata.for_query_tile( - q, extra_width, staging_dtype=attn.staging_dtype - ) + metadata = CSA2Metadata.for_query_tile(q, extra_width, staging_dtype=attn.staging_dtype) else: - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=count) + source = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=count) source._num_contexts = 2 source._num_ctx_tokens = source._num_tokens = count source.csa2_num_context_requests = 2 @@ -1314,8 +1302,6 @@ def test_default_staging_follows_the_backend_that_can_serve_it(kv_dtype): @skip_pre_blackwell @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") def test_native_fp8_rejects_incompatible_options(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - # Packed attention reads persistent rows directly and has no FP8 staging. with pytest.raises(ValueError): _backend(64, kv_cache_dtype="fp8", use_packed=True, fp8_staging=True) @@ -1326,9 +1312,7 @@ def test_native_fp8_rejects_incompatible_options(): valid = torch.ones(1, 128, device="cuda", dtype=torch.bool) args = _inputs(q, swa, None, valid, None, torch.zeros(64, device="cuda")) with pytest.raises(ValueError): - attn.forward( - q.flatten(1), None, None, CSA2TrtllmMetadata.for_query_tile(q, 0), forward_args=args - ) + attn.forward(q.flatten(1), None, None, CSA2Metadata.for_query_tile(q, 0), forward_args=args) @skip_pre_blackwell @@ -1345,10 +1329,7 @@ def test_shared_native_context_matches_selected_rows( AttentionInputType, ) from tensorrt_llm._torch.attention.backends.sparse.csa2 import kernel - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import ( - CSA2SharedKVPlan, - CSA2TrtllmMetadata, - ) + from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2SharedKVPlan from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2BackendForwardArgs from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( gather_rows, @@ -1410,7 +1391,7 @@ def observe(*args, **kwargs): monkeypatch.setattr(kernel, "stage_shared_rows", observe) outputs = [] for shared in (False, True): - metadata = CSA2TrtllmMetadata.for_query_tile( + metadata = CSA2Metadata.for_query_tile( q, width, staging_dtype=attn.staging_dtype, @@ -1532,6 +1513,7 @@ def phase(n, contexts, graph_reserved=False): metadata = SimpleNamespace( num_contexts=contexts, shared_plan=True, _csa2_graph_reserved=graph_reserved ) + metadata.ext = lambda cls: metadata if contexts: args = AttentionForwardArgs(attention_input_type=AttentionInputType.context_only) return helper.forward_context(query, metadata, args) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py b/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py index fe0affaa2ee3..62df853eb95c 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py @@ -7,11 +7,11 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import ( CSA2CacheManager, CSA2CacheRole, ) -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( read_index_rows, @@ -401,7 +401,7 @@ def _request(request_id, tokens): def _metadata(manager, request, metadata=None): - metadata = metadata or CSA2TrtllmMetadata( + metadata = metadata or CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=512, kv_cache_manager=manager ) metadata.request_ids = [request.py_request_id] @@ -416,7 +416,7 @@ def _metadata(manager, request, metadata=None): @pytest.mark.cpu_only -def test_swa_publication_is_scoped_transaction_with_exact_request_and_destination(): +def test_swa_publication_matches_exact_request_and_destination(): manager = CSA2CacheManager.__new__(CSA2CacheManager) manager._swa_publication_token = manager._swa_publication = None manager.pp_layers = [0, 1] @@ -442,34 +442,62 @@ def compute(request_ids, num_contexts): manager.compute_batch_page_tables = compute destination = torch.full((2, 4, 2, 4), 99, dtype=torch.int32) ids = [3, 3, 1] # Repeated padding IDs retain packed row order. - with manager._swa_publication_scope() as token: + # Each step: the core prepares the batch tables, copies the block offsets + # (which publishes the SWA tables) and CSA2's prepare consumes them. + manager.prepare_batch_tables(ids, 2) + manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) + token = manager.consume_swa_publication() + assert token is not None + device = manager._get_swa_publication(token, destination, ids, 2, 3) + torch.testing.assert_close(manager._batch_page_tables[:, :, :3], device, atol=0, rtol=0) + assert device.stride(-2) == 8 and torch.all(destination[:, 3] == -1) + assert manager._get_swa_publication(token, destination, ids, 2, 5) is None + assert manager._get_swa_publication(token, destination, [1, 3, 3], 2, 3) is None + assert manager._get_swa_publication(token, destination, ids, 1, 3) is None + assert manager._get_swa_publication(token, destination.clone(), ids, 2, 3) is None + state["fail"] = True + with pytest.raises(ValueError, match="conversion failed"): manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) - device = manager._get_swa_publication(token, destination, ids, 2, 3) - torch.testing.assert_close(manager._batch_page_tables[:, :, :3], device, atol=0, rtol=0) - assert device.stride(-2) == 8 and torch.all(destination[:, 3] == -1) - assert manager._get_swa_publication(token, destination, ids, 2, 5) is None - assert manager._get_swa_publication(token, destination, [1, 3, 3], 2, 3) is None - assert manager._get_swa_publication(token, destination, ids, 1, 3) is None - assert manager._get_swa_publication(token, destination.clone(), ids, 2, 3) is None - state["fail"] = True - with pytest.raises(ValueError, match="conversion failed"): - manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) - assert manager._get_swa_publication(token, destination, ids, 2, 3) is None - assert manager._swa_publication_token is manager._swa_publication is None + assert manager._get_swa_publication(token, destination, ids, 2, 3) is None + assert manager.consume_swa_publication() is None state.update(fail=False, offset=100) - with manager._swa_publication_scope() as fresh: - manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) - assert manager._get_swa_publication(token, destination, ids, 2, 3) is None - assert manager._get_swa_publication(fresh, destination, ids, 2, 3)[0, 0, 0] == 103 - destination.set_(torch.empty_like(destination)) - assert manager._get_swa_publication(fresh, destination, ids, 2, 3) is None - with pytest.raises(RuntimeError, match="later preparation failed"): - with manager._swa_publication_scope(): - manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) - raise RuntimeError("later preparation failed") + manager.prepare_batch_tables(ids, 2) + manager.copy_batch_block_offsets(destination, ids, 1, 2, 3) + fresh = manager.consume_swa_publication() + assert manager._get_swa_publication(token, destination, ids, 2, 3) is None + assert manager._get_swa_publication(fresh, destination, ids, 2, 3)[0, 0, 0] == 103 + manager.close_swa_publication(fresh) + assert manager._get_swa_publication(fresh, destination, ids, 2, 3) is None assert manager._swa_publication_token is manager._swa_publication is None +def test_unconsumed_swa_publication_does_not_reach_the_next_step(): + manager = CSA2CacheManager.__new__(CSA2CacheManager) + manager._swa_publication_token = manager._swa_publication = None + manager.pp_layers = [0] + manager.max_blocks_per_seq = 2 + manager.has_swa_cache = lambda layer: True + + def compute(request_ids, num_contexts): + manager._batch_page_tables = torch.zeros((1, len(request_ids), 2), dtype=torch.int32) + + manager.compute_batch_page_tables = compute + destination = torch.zeros((1, 2, 2, 2), dtype=torch.int32) + # A step publishes its SWA tables but no CSA2 prepare consumes them. + manager.prepare_batch_tables([0, 1], 0) + manager.copy_batch_block_offsets(destination, [0, 1], 1, 0, 2) + stale = manager.consume_swa_publication() + assert stale is not None + # The next step's preparation drops it before its own copy. + manager.prepare_batch_tables([0, 1], 0) + assert manager.consume_swa_publication() is None + assert manager._get_swa_publication(stale, destination, [0, 1], 0, 2) is None + # Its own copy publishes afresh. + manager.copy_batch_block_offsets(destination, [0, 1], 1, 0, 2) + fresh = manager.consume_swa_publication() + assert fresh is not None and fresh is not stale + + # Embedded DSpark linear verification replayed with the executor protocol. DRAFT_LEN = 5 @@ -494,7 +522,7 @@ def _spec_manager(block): def _spec_metadata(manager): - return CSA2TrtllmMetadata( + return CSA2TrtllmAttention.Metadata( max_num_requests=manager.max_batch_size, max_num_tokens=manager.max_num_tokens, kv_cache_manager=manager, diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_context_only_metadata.py b/tests/unittest/_torch/attention/sparse/csa2/test_context_only_metadata.py index 02da5bd7eb48..ff4119b065de 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_context_only_metadata.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_context_only_metadata.py @@ -6,11 +6,11 @@ import torch from utils.util import skip_pre_blackwell +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import ( CSA2CacheManager, CSA2CacheRole, ) -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest @@ -58,7 +58,7 @@ def test_context_metadata_omits_decoder_swa_and_preserves_boundary_global(length assert manager.prepare_context(request) assert manager.resize_context(request, length) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=8192, kv_cache_manager=manager ) metadata.request_ids = [71] @@ -73,7 +73,7 @@ def test_context_metadata_omits_decoder_swa_and_preserves_boundary_global(length for direct in (False, True): if direct: metadata.prepare_csa2() - metadata.on_update_kv_lens() + metadata._refresh_live_endpoints() metadata._ensure_swa_slots() assert set(metadata.csa2_swa_indices) == set(range(40)) assert set(metadata._csa2_swa_page_tables) == set(range(20)) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_indexer.py b/tests/unittest/_torch/attention/sparse/csa2/test_indexer.py index 09197c391c13..fe1cc5dc9b5f 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_indexer.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_indexer.py @@ -6,6 +6,7 @@ import torch import torch.nn.functional as F +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager from tensorrt_llm._torch.attention.backends.sparse.csa2.compressor import CSA2CompressionBatch from tensorrt_llm._torch.attention.backends.sparse.csa2.indexer import ( @@ -14,7 +15,7 @@ _ChunkInputs, _QueryChunk, ) -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import ( CSA2ForwardState, CSA2Layout, @@ -862,7 +863,7 @@ def test_native_paged_owner_indexer_and_graph(monkeypatch, ratio): # One request carries two verification queries; declare that arena width. manager.max_total_draft_tokens = 1 q, _, weights = _projected(3) - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=3) + metadata = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=3) metadata.kv_cache_manager = manager metadata.mapping = Mapping() metadata.is_cuda_graph = True @@ -883,7 +884,7 @@ def test_native_paged_owner_indexer_and_graph(monkeypatch, ratio): metadata.csa2_candidates = {} metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} - metadata.indexer_max_chunk_size = 16 + metadata.csa2_indexer_max_chunk_size = 16 metadata.indexer_q_split_threshold = -1 state = CSA2ForwardState( metadata=metadata, @@ -990,7 +991,9 @@ def test_multi_rank_candidate_publication_and_reindex(count, chunk_size, mixed, manager = _StridedOwner(layout, keys) total = count + int(mixed) q, _, weights = _projected(total) - metadata = CSA2TrtllmMetadata(max_num_requests=1 + int(mixed), max_num_tokens=total) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1 + int(mixed), max_num_tokens=total + ) metadata.kv_cache_manager = manager metadata.mapping = Mapping(world_size=world_size, rank=mpi_rank(), tp_size=world_size) metadata.csa2_kv_sources = {0: 0, 1: 0} @@ -1020,7 +1023,7 @@ def test_multi_rank_candidate_publication_and_reindex(count, chunk_size, mixed, metadata.csa2_candidates = {} metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} - metadata.indexer_max_chunk_size = chunk_size + metadata.csa2_indexer_max_chunk_size = chunk_size metadata.indexer_q_split_threshold = 0 state = CSA2ForwardState( metadata=metadata, @@ -1090,7 +1093,7 @@ def _single_request_state(heads=8, two_owners=False): 1: _index_pages_from_rows(pack_rows(-keys, "index")), } manager.get_index_pages = lambda layer: manager.owner_buffers[layout.layer(layer).kv_source] - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=1) + metadata = CSA2TrtllmAttention.Metadata(max_num_requests=1, max_num_tokens=1) metadata.kv_cache_manager = manager metadata.mapping = Mapping() metadata.csa2_kv_sources = {i: i for i in layout.kv_source_layer_ids} @@ -1117,7 +1120,7 @@ def _single_request_state(heads=8, two_owners=False): metadata.csa2_candidates = {} metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} - metadata.indexer_max_chunk_size = 16 + metadata.csa2_indexer_max_chunk_size = 16 metadata.indexer_q_split_threshold = -1 state = CSA2ForwardState( metadata=metadata, @@ -1160,7 +1163,7 @@ def gather(owner, slots): monkeypatch.setattr(manager, "gather_indexer_keys", gather) # Exercise the bounded fallback independently of native head coverage. - monkeypatch.setattr(metadata, "prepare_indexer", lambda layer: None) + monkeypatch.setattr(metadata.ext(CSA2Metadata), "prepare_indexer", lambda layer: None) monkeypatch.setattr( indexer, "_call_paged_mqa_logits", @@ -1283,7 +1286,7 @@ def test_paged_descriptors_refresh_when_the_manager_changes(): layout, manager, metadata, state = _single_request_state() metadata.is_cuda_graph = True metadata.csa2_visible_lengths[0].fill_(96) - assert metadata.prepare_indexer(0) is metadata + assert metadata.prepare_indexer(0) is metadata.ext(CSA2Metadata) assert metadata.csa2_indexer_block_table.tolist() == [[0, 1]] keys = torch.ones(512, 128, device="cuda", dtype=torch.bfloat16) draft = _StridedOwner(layout, pack_rows(keys, "index")) @@ -1292,7 +1295,7 @@ def test_paged_descriptors_refresh_when_the_manager_changes(): torch.tensor([[3, 2]], device="cuda", dtype=torch.int32) ) # Same forward serial, different manager: descriptors must follow the draft. - assert metadata.prepare_indexer(0) is metadata + assert metadata.prepare_indexer(0) is metadata.ext(CSA2Metadata) assert metadata.csa2_indexer_k_cache.data_ptr() == draft.index_pages.data_ptr() assert metadata.csa2_indexer_block_table.tolist() == [[3, 2]] @@ -1351,7 +1354,7 @@ def _temporal_gvr_case(monkeypatch, emission, *, page_holes=False, candidate_sou ) manager.epoch = 1 manager.request_epoch = lambda request: manager.epoch - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=1) + metadata = CSA2TrtllmAttention.Metadata(max_num_requests=1, max_num_tokens=1) metadata.kv_cache_manager = manager metadata.mapping = Mapping() metadata.request_ids = [13] @@ -1377,7 +1380,7 @@ def _temporal_gvr_case(monkeypatch, emission, *, page_holes=False, candidate_sou metadata.csa2_candidates = {} metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} - metadata.indexer_max_chunk_size = 16 + metadata.csa2_indexer_max_chunk_size = 16 metadata.indexer_q_split_threshold = -1 metadata._csa2_forward_serial = 0 state = CSA2ForwardState( @@ -1606,7 +1609,6 @@ def test_live_acceptance_graph_refreshes_native_paged_indexer(monkeypatch, query metadata.csa2_request_lengths = [query_width] metadata.csa2_request_start_positions = [127] device = torch.device("cuda") - metadata._csa2_ready_for_kv_update = True metadata._csa2_query_base = torch.tensor([127], dtype=torch.int32, device=device) metadata._csa2_query_lengths = torch.tensor([query_width], dtype=torch.int32, device=device) metadata._csa2_query_offsets = torch.arange(query_width, dtype=torch.int32, device=device) @@ -1625,7 +1627,11 @@ def test_live_acceptance_graph_refreshes_native_paged_indexer(monkeypatch, query for owner in (0, 1) } metadata._csa2_source_geometry = { - owner: (metadata._csa2_query_base, metadata._csa2_query_lengths) for owner in (0, 1) + owner: ( + metadata._csa2_query_base, + metadata._csa2_query_lengths, + ) + for owner in (0, 1) } metadata.csa2_main_write_slots = { owner: torch.empty(query_width, dtype=torch.int64, device=device) for owner in (0, 1) @@ -1661,7 +1667,7 @@ def test_live_acceptance_graph_refreshes_native_paged_indexer(monkeypatch, query ) def run(): - metadata.on_update_kv_lens() + metadata._refresh_live_endpoints() metadata._ensure_swa_slots() # As entering the first layer does. for owner in (0, 1): kept[owner][0].copy_(metadata.csa2_swa_write_slots[owner]) @@ -1842,7 +1848,7 @@ def _publish_manual_candidates(metadata, layer, block_ids, block_size=8): ranges = metadata.csa2_request_query_ranges decode_start = ranges[contexts][0] if contexts < len(ranges) else count rows = (requests[decode_start:] - contexts).int() - existing = getattr(metadata, "csa2_decode_row_requests", None) + existing = getattr(metadata.ext(CSA2Metadata), "csa2_decode_row_requests", None) # Captured graphs read this map by address as well: update it in place. if existing is not None and existing.shape == rows.shape: existing.copy_(rows) @@ -1864,7 +1870,9 @@ def _candidate_decode_fixture(*, mixed=False, random=False): manager = _StridedOwner(layout, pack_rows(keys, "index")) manager.max_total_draft_tokens = 1 count = 4 if mixed else 3 - metadata = CSA2TrtllmMetadata(max_num_requests=3 if mixed else 2, max_num_tokens=count) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=3 if mixed else 2, max_num_tokens=count + ) metadata.kv_cache_manager = manager metadata.mapping = Mapping() metadata.is_cuda_graph = True @@ -1889,7 +1897,7 @@ def _candidate_decode_fixture(*, mixed=False, random=False): metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} _publish_manual_candidates(metadata, 0, _MANUAL_BLOCKS) - metadata.indexer_max_chunk_size = 16 + metadata.csa2_indexer_max_chunk_size = 16 metadata.indexer_q_split_threshold = -1 q = torch.zeros(count, 32, 128, dtype=torch.bfloat16, device="cuda") q[:, :9, :9] = torch.eye(9, dtype=torch.bfloat16, device="cuda") @@ -2022,7 +2030,9 @@ def test_candidate_decode_batched_exact_and_mixed(monkeypatch, mixed): # Without paged sparse descriptors the decode rows fall back to the # per-query candidate gathers without row-offset loss. with monkeypatch.context() as context: - context.setattr(state.metadata, "prepare_sparse_indexer", lambda *a: None) + context.setattr( + state.metadata.ext(CSA2Metadata), "prepare_sparse_indexer", lambda *a: None + ) fallback, fallback_scores, fallback_calls = _candidate_decode_run( indexer, state, monkeypatch, old=False ) @@ -2098,7 +2108,10 @@ def test_candidate_batched_random_logits(monkeypatch, phase): # Existing native selection tests allow only 0.005 absolute cutoff error. torch.testing.assert_close(new_scores[valid], scores[valid], atol=0.005, rtol=2**-7) _check_candidate_multiset( - actual, scores, state.metadata.csa2_candidates[0], 0.005 + _bf16_tolerance(scores) + actual, + scores, + state.metadata.csa2_candidates[0], + 0.005 + _bf16_tolerance(scores), ) separated = gaps > 2 * difference.abs().amax(1) torch.testing.assert_close(actual[separated], old[separated], atol=0, rtol=0) @@ -2442,7 +2455,7 @@ def test_sparse_candidate_paged_decode_and_graph(monkeypatch): manager = _StridedOwner(layout, packed_keys) manager.max_total_draft_tokens = 1 q, _, weights = _projected(3) - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=3) + metadata = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=3) metadata.kv_cache_manager = manager metadata.mapping = Mapping() metadata.is_cuda_graph = True @@ -2463,7 +2476,7 @@ def test_sparse_candidate_paged_decode_and_graph(monkeypatch): metadata.csa2_candidates = {} metadata.csa2_candidate_blocks = {} metadata.csa2_candidate_counts = {} - metadata.indexer_max_chunk_size = 16 + metadata.csa2_indexer_max_chunk_size = 16 metadata.indexer_q_split_threshold = -1 state = CSA2ForwardState( metadata=metadata, diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_indexer_descriptors.py b/tests/unittest/_torch/attention/sparse/csa2/test_indexer_descriptors.py index 94aea2804df4..25716951a6dd 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_indexer_descriptors.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_indexer_descriptors.py @@ -4,17 +4,14 @@ from __future__ import annotations -from contextlib import nullcontext -from copy import copy -from types import SimpleNamespace from typing import TYPE_CHECKING import pytest import torch -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import INDEX_PAGE_ROWS -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from . import test_metadata from .test_metadata import _prepare_decode_metadata @@ -153,8 +150,8 @@ def test_indexer_descriptors_follow_live_graph_inputs( ) -> None: """Replay changed contents against eager and scalar references with exact integer parity.""" monkeypatch.setattr("tensorrt_llm._torch.utils.is_piecewise_running", lambda: False) - eager = CSA2TrtllmMetadata._fill_indexer_descriptors - captured = CSA2TrtllmMetadata._fill_indexer_descriptors + eager = CSA2Metadata._fill_indexer_descriptors + captured = CSA2Metadata._fill_indexer_descriptors host_inputs = _inputs(query_count, request_count, source_pages, page_capacity, 0) stream = torch.cuda.Stream() stream.wait_stream(torch.cuda.current_stream()) @@ -202,67 +199,30 @@ def test_indexer_descriptors_follow_live_graph_inputs( assert changed, "The graph must observe changing descriptor inputs" -@pytest.mark.cpu_only -def test_graph_clone_post_init_clears_decode_deferral_state( - monkeypatch: pytest.MonkeyPatch, -) -> None: - metadata = object.__new__(CSA2TrtllmMetadata) - metadata._csa2_deferred_decode_outputs = True - clone = copy(metadata) - monkeypatch.setattr(TrtllmAttentionMetadata, "__post_init__", lambda self: None) - clone.__post_init__() - assert not clone._csa2_deferred_decode_outputs - assert metadata._csa2_deferred_decode_outputs - - -@pytest.mark.cpu_only -@pytest.mark.parametrize("entrypoint", ["prepare", "prepare_csa2"]) -def test_failed_prepare_clears_decode_deferral_state( - monkeypatch: pytest.MonkeyPatch, entrypoint: str -) -> None: - metadata = object.__new__(CSA2TrtllmMetadata) - metadata._csa2_deferred_decode_outputs = True - metadata.kv_cache_manager = SimpleNamespace() - metadata.is_cuda_graph = False - metadata.kv_lens_cuda = torch.empty(0) - - def fail_prepare() -> None: - raise ValueError("injected prepare failure") - - monkeypatch.setattr(torch.cuda, "device", lambda _: nullcontext()) - monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", fail_prepare) - error = ValueError if entrypoint == "prepare" else TypeError - message = "injected prepare failure" if entrypoint == "prepare" else "CSA2CacheManager" - with pytest.raises(error, match=message): - getattr(metadata, entrypoint)() - assert not metadata._csa2_deferred_decode_outputs - - @skip_pre_blackwell @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() -def test_selected_decode_prepare_controls_indexer_dispatch( +def test_decode_prepare_refreshes_each_owner_once( manager_requests: tuple[CSA2CacheManager, list[LlmRequest]], monkeypatch: pytest.MonkeyPatch, ) -> None: manager, requests = manager_requests - base = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) - # A shallow graph clone must discard any eligibility inherited from its source. - base._csa2_deferred_decode_outputs = True + base = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) metadata = base.create_cuda_graph_metadata(2) - assert not metadata._csa2_deferred_decode_outputs - assert base._csa2_deferred_decode_outputs + csa2 = metadata.ext(CSA2Metadata) starts = [127, 128] calls = [] - eager = CSA2TrtllmMetadata._fill_indexer_descriptors + eager = CSA2Metadata._fill_indexer_descriptors def record_refresh(*args: torch.Tensor | int) -> None: - calls.append(metadata._csa2_deferred_decode_outputs) + calls.append(args) return eager(*args) # The same fused dispatcher serves every preparation mode. Graph parity - # is checked above; isolate per-owner reuse and preparation state here. - monkeypatch.setattr(metadata, "_fill_indexer_descriptors", record_refresh) + # is checked above; isolate per-owner reuse here. + monkeypatch.setattr(csa2, "_fill_indexer_descriptors", record_refresh) monkeypatch.setattr("tensorrt_llm._torch.utils.is_piecewise_running", lambda: True) monkeypatch.setattr( "tensorrt_llm.deep_gemm.get_paged_mqa_logits_metadata", @@ -271,29 +231,15 @@ def record_refresh(*args: torch.Tensor | int) -> None: ), ) - _prepare_decode_metadata(metadata, requests, starts, deferred=True) - assert metadata._csa2_deferred_decode_outputs - assert not metadata._csa2_defer_decode_outputs - metadata.on_update_kv_lens() - assert metadata.prepare_indexer(1) is metadata - assert calls == [True] + _prepare_decode_metadata(metadata, requests, starts) + assert metadata.prepare_indexer(1) is csa2 + assert len(calls) == 1 # Layers 1 and 2 share an owner. One serial refresh serves both layers. metadata.prepare_indexer(1) metadata.prepare_indexer(2) - assert calls == [True] + assert len(calls) == 1 + # Every prepare starts a new serial. _prepare_decode_metadata(metadata, requests, starts) - assert not metadata._csa2_deferred_decode_outputs - metadata.on_update_kv_lens() - metadata.prepare_indexer(1) - assert calls == [True, False] - - _prepare_decode_metadata(metadata, requests, starts, deferred=True) - assert metadata._csa2_deferred_decode_outputs - metadata.set_source_batch([1, 1], starts) - _prepare_decode_metadata(metadata, requests, starts, deferred=True) - assert not hasattr(metadata, "_csa2_source_batch") - assert not metadata._csa2_deferred_decode_outputs - metadata.on_update_kv_lens() metadata.prepare_indexer(1) - assert calls == [True, False, False] + assert len(calls) == 2 diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_kernel.py b/tests/unittest/_torch/attention/sparse/csa2/test_kernel.py index 71a1f1803c1d..2931502382d4 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_kernel.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_kernel.py @@ -14,6 +14,7 @@ packed_sparse_attention, quantize_scatter_rows, ) +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( INDEX_PAGE_BYTES, INDEX_PAGE_ROWS, @@ -673,12 +674,11 @@ def test_quantization_cuda_graph_changed_values(): def _stage_fixture(queries, swa_width, main_width, *, empty_pool=False, extra_width=None): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2BackendForwardArgs torch.manual_seed(711) q = torch.empty(queries, 1, 512, dtype=torch.bfloat16, device="cuda") - metadata = CSA2TrtllmMetadata.for_query_tile( + metadata = CSA2Metadata.for_query_tile( q, (main_width or 0) if extra_width is None else extra_width ) rows = 0 if empty_pool else 23 @@ -726,7 +726,10 @@ def _stage_oracle(metadata, inputs): packed = rows.gather(1, order[..., None].expand(-1, -1, 512)) counts = valid.sum(1, dtype=torch.int32) packed = torch.where((positions < counts[:, None])[..., None], packed, 0) - primary, extra = torch.zeros_like(metadata.swa_pool), torch.zeros_like(metadata.extra_pool) + primary, extra = ( + torch.zeros_like(metadata.swa_pool), + torch.zeros_like(metadata.extra_pool), + ) primary[:, : min(rows.shape[1], 128)] = packed[:, :128] if rows.shape[1] > 128: extra[:, : rows.shape[1] - 128] = packed[:, 128:] @@ -817,7 +820,13 @@ def test_staging_group_reuse_validates_each_layers_own_slots(): } # The first layer of the group resolves every column, so it tags all of them. stage_selected_rows( - inputs.swa_pool, resolved, inputs.swa_pool, None, metadata, scratch, compact=True + inputs.swa_pool, + resolved, + inputs.swa_pool, + None, + metadata.ext(CSA2Metadata), + scratch, + compact=True, ) assert scratch["counts"].tolist() == [width] * queries # The later layer's private pool starts one row into a sentinel buffer, so the @@ -831,7 +840,7 @@ def test_staging_group_reuse_validates_each_layers_own_slots(): torch.full((queries, width), -1, device="cuda", dtype=torch.int64), sentinel[1:], None, - metadata, + metadata.ext(CSA2Metadata), scratch, compact=False, ) @@ -847,7 +856,7 @@ def test_native_staging_graph_refresh(): metadata, inputs = _stage_fixture(8, 128, 17) metadata.stage_selected(inputs) tensor_names = ("swa_pool", "extra_pool", "prepared_indices", "prepared_lens") - pointers = [getattr(metadata, name).data_ptr() for name in tensor_names] + pointers = [getattr(metadata.ext(CSA2Metadata), name).data_ptr() for name in tensor_names] retained = metadata.get_workspace_bytes() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): @@ -863,7 +872,9 @@ def test_native_staging_graph_refresh(): metadata.prepared_counter.fill_(19) graph.replay() _check_stage(metadata, expected) - assert pointers == [getattr(metadata, name).data_ptr() for name in tensor_names] + assert pointers == [ + getattr(metadata.ext(CSA2Metadata), name).data_ptr() for name in tensor_names + ] assert retained == metadata.get_workspace_bytes() @@ -931,7 +942,6 @@ def test_fused_store_changed_row_counts(cache_format, dim): @torch.inference_mode() def test_native_fp8_staging_exact_bytes(dequant, q_dequant, monkeypatch): from tensorrt_llm._torch.attention.backends.sparse.csa2 import kernel - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() control, inputs = _stage_fixture(3, 128, 17) @@ -951,7 +961,7 @@ def test_native_fp8_staging_exact_bytes(dequant, q_dequant, monkeypatch): inputs.topk_indices[2, :2] = 2 expected = _stage_oracle(control, inputs) q = torch.zeros(3, 64, 512, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) dq = torch.tensor([dequant], device="cuda", dtype=torch.float32) q_dq = torch.tensor([q_dequant], device="cuda", dtype=torch.float32) multiplier = dq.reciprocal() @@ -1030,7 +1040,6 @@ def _stage_scale_reference(inputs): @torch.inference_mode() def test_stage_kv_scale_derivation(case, minimum): """The derived staging scale covers every selected row without saturating.""" - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() _, inputs = _stage_fixture(3, 128, 17) @@ -1052,7 +1061,7 @@ def test_stage_kv_scale_derivation(case, minimum): inputs.swa_pool[2, 512:].fill_(254) # 2**127, past every clamp inputs.swa_indices[2, 0] = 2 q = torch.zeros(3, 64, 512, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) quant, dequant = metadata.derive_stage_kv_scales(inputs, minimum=minimum, maximum=126) assert (quant, dequant) == metadata.stage_kv_scales scale = dequant.item() @@ -1093,7 +1102,6 @@ def test_stage_kv_scale_respects_q_headroom(amax, expected): its codes land at ``scale * amax(|Q|)``: a scale wide enough to rescue the KV rows must still leave Q inside the E4M3 range. """ - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() _, inputs = _stage_fixture(3, 128, 17) @@ -1101,7 +1109,7 @@ def test_stage_kv_scale_respects_q_headroom(amax, expected): inputs.swa_pool[2, 512:].fill_(254) inputs.swa_indices[2, 0] = 2 q = torch.full((3, 64, 512), amax, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) quant, dequant = metadata.derive_stage_kv_scales(inputs, q=q) assert dequant.item() == expected assert quant.item() == 1.0 / expected @@ -1114,7 +1122,6 @@ def test_stage_kv_scale_respects_q_headroom(amax, expected): @torch.inference_mode() def test_stage_kv_scale_rescues_saturated_rows(): """Rows decoding past 448 survive staging instead of clamping at the E4M3 max.""" - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() control, inputs = _stage_fixture(4, 128, 17) @@ -1130,7 +1137,7 @@ def test_stage_kv_scale_rescues_saturated_rows(): ) expected = _stage_oracle(control, inputs) q = torch.zeros(4, 64, 512, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) unit = metadata.native_unit_scale metadata.stage_selected(inputs, kv_scale_orig_quant=unit, kv_scale_quant_orig=unit) clamped = [pool.float().clone() for pool in (metadata.swa_pool, metadata.extra_pool)] @@ -1171,7 +1178,6 @@ def test_stage_kv_scale_respects_q_resolution(amax, expected): bound therefore holds Q's own amax at unit magnitude or above, which leaves it the whole E4M3 normal range below one plus the subnormals under that. """ - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() _, inputs = _stage_fixture(3, 128, 17) @@ -1179,7 +1185,7 @@ def test_stage_kv_scale_respects_q_resolution(amax, expected): inputs.swa_pool[2, 512:].fill_(254) inputs.swa_indices[2, 0] = 2 q = torch.full((3, 64, 512), amax, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) quant, dequant = metadata.derive_stage_kv_scales(inputs, q=q, q_shared=True) assert dequant.item() == expected assert quant.item() == 1.0 / expected @@ -1203,7 +1209,6 @@ def test_stage_kv_scale_rescues_context_rows(): from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import ( _CONTEXT_STAGE_SCALE_MAX_EXPONENT, ) - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() control, inputs = _stage_fixture(4, 128, 17) @@ -1222,7 +1227,7 @@ def test_stage_kv_scale_rescues_context_rows(): inputs.main_pool[0, 256:].fill_(126) expected = _stage_oracle(control, inputs) q = torch.full((4, 64, 512), 64.0, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) + metadata = CSA2Metadata.for_query_tile(q, 17, staging_dtype=torch.float8_e4m3fn) unit = metadata.native_unit_scale metadata.stage_selected(inputs, kv_scale_orig_quant=unit, kv_scale_quant_orig=unit) clamped = [pool.float().clone() for pool in (metadata.swa_pool, metadata.extra_pool)] @@ -1253,7 +1258,6 @@ def test_stage_kv_scale_rescues_context_rows(): def test_unfused_fp8_staging_matches_fused_bytes(monkeypatch): """The eager staging fallback reproduces the fused route byte for byte.""" from tensorrt_llm._torch.attention.backends.sparse.csa2 import kernel - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata _sm100() calls = [] @@ -1272,9 +1276,7 @@ def stage(*args, **kwargs): inputs.main_pool[1, 256:].fill_(126) calls.clear() q = torch.zeros(2, 64, 512, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile( - q, extra_width, staging_dtype=torch.float8_e4m3fn - ) + metadata = CSA2Metadata.for_query_tile(q, extra_width, staging_dtype=torch.float8_e4m3fn) quant, dequant = metadata.derive_stage_kv_scales(inputs) assert dequant.item() == 8.0 # 2688 mapped onto 448. metadata.mla_bmm1_scale.fill_(float("nan")) @@ -1347,8 +1349,7 @@ def strided(values): ) bank = torch.full((25, 512), 7, device="cuda", dtype=dtype) metadata = SimpleNamespace( - num_tokens=5, - num_sparse_topk=128, + metadata=SimpleNamespace(num_tokens=5, num_sparse_topk=128), shared_pool=bank, _csa2_shared_selected=torch.empty(25, device="cuda", dtype=torch.int32), prepared_indices=torch.empty(5, 128, device="cuda", dtype=torch.int32), @@ -1357,6 +1358,7 @@ def strided(values): mla_bmm1_scale=torch.empty(2, device="cuda"), mla_bmm2_scale=torch.empty(1, device="cuda"), ) + metadata.ext = lambda cls: metadata unit = torch.ones(1, device="cuda") stage_shared_rows( swa, @@ -1460,7 +1462,10 @@ def strided(values): 0 ] torch.testing.assert_close( - metadata.shared_pool.view(torch.uint8), expected_bank.view(torch.uint8), atol=0, rtol=0 + metadata.shared_pool.view(torch.uint8), + expected_bank.view(torch.uint8), + atol=0, + rtol=0, ) @@ -1481,8 +1486,6 @@ def _glue(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_owner_refresh_kernel_matches_reference(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - kernel = _glue() torch.manual_seed(1201) request_count = 3 @@ -1526,7 +1529,7 @@ def test_owner_refresh_kernel_matches_reference(): for source_base, source_lengths, pages, slots, compressed, ratio, page_size, *batch in owners: outputs = [t for t in (slots, compressed, *batch) if t is not None] expected = [torch.empty_like(t) for t in outputs] - CSA2TrtllmMetadata._refresh_owner_slots( + CSA2Metadata._refresh_owner_slots( source_base, source_lengths, delta, @@ -1715,8 +1718,7 @@ def strided_pages(values: list[list[int]]) -> torch.Tensor: ) bank = torch.full((main_offsets[-1], 512), 7, device="cuda", dtype=dtype) metadata = SimpleNamespace( - num_tokens=len(query_requests), - num_sparse_topk=128, + metadata=SimpleNamespace(num_tokens=len(query_requests), num_sparse_topk=128), shared_pool=bank, _csa2_shared_selected=torch.empty(bank.shape[0], device="cuda", dtype=torch.int32), prepared_indices=torch.empty(len(query_requests), 128, device="cuda", dtype=torch.int32), @@ -1725,6 +1727,7 @@ def strided_pages(values: list[list[int]]) -> torch.Tensor: mla_bmm1_scale=torch.empty(2, device="cuda"), mla_bmm2_scale=torch.empty(1, device="cuda"), ) + metadata.ext = lambda cls: metadata dequant_scale = torch.tensor([0.3], device="cuda", dtype=torch.float32) stage_shared_rows( swa, @@ -1821,8 +1824,7 @@ def test_shared_active_rows_reuse_request_count_specialization( ) bank = torch.empty(1 + 2 * domain_rows, 512, device="cuda", dtype=dtype) metadata = SimpleNamespace( - num_tokens=queries, - num_sparse_topk=128, + metadata=SimpleNamespace(num_tokens=queries, num_sparse_topk=128), shared_pool=bank, _csa2_shared_selected=torch.empty(bank.shape[0], device="cuda", dtype=torch.int32), prepared_indices=torch.empty(queries, 128, device="cuda", dtype=torch.int32), @@ -1831,6 +1833,7 @@ def test_shared_active_rows_reuse_request_count_specialization( mla_bmm1_scale=torch.empty(2, device="cuda"), mla_bmm2_scale=torch.empty(1, device="cuda"), ) + metadata.ext = lambda cls: metadata swa_slots = torch.empty(queries, 1, device="cuda", dtype=torch.int64) main_slots = torch.empty(queries, 2, device="cuda", dtype=torch.int64) main_logical = torch.empty_like(main_slots) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py b/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py index 2b9d71afc0d7..ca1c957279a6 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py @@ -9,6 +9,7 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager from tensorrt_llm._torch.attention.backends.sparse.csa2.compressor import CSA2CompressionBatch from tensorrt_llm._torch.attention.backends.sparse.csa2.decoder_replay import ( @@ -17,17 +18,28 @@ exit_decoder_replay, plan_decoder_replay, ) -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import pack_rows -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.mapping import Mapping -from ._utils import _FakeMetadata, _page_metadata +from ._utils import _bare_csa2_metadata, _FakeMetadata, _page_metadata + + +def _assign(metadata, **values) -> None: + """Set metadata fields on the extension that owns each (the metadata reads them).""" + for name, value in values.items(): + owner = next( + extension + for extension in metadata.extensions + if name in type(extension).declared_fields() + ) + setattr(owner, name, value) + skip_pre_blackwell = pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.get_device_capability() < (10, 0), @@ -41,7 +53,7 @@ def test_decoder_swa_slots_skip_reclaimed_encoder_pages(device): if device == "cuda" and not (torch.cuda.is_available() and kernel.dsl_available()): pytest.skip("CuTe DSL SWA refresh requires CUDA") - metadata = object.__new__(CSA2TrtllmMetadata) + metadata = _bare_csa2_metadata() metadata.is_cuda_graph = False metadata.kv_lens_cuda = torch.zeros(1, device=device) metadata.kv_cache_manager = SimpleNamespace( @@ -171,10 +183,11 @@ def _swa(metadata, layer): @torch.inference_mode() def test_manager_metadata_scratch_and_source_capacity(manager_requests): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheRole - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) _prepare(metadata, requests, [0, 0], [257, 257]) _assert_swa_cpu_reference(metadata, manager, requests, [0, 0], [257, 257], [0, 0]) reads, writes, _ = _swa(metadata, 1) @@ -222,21 +235,23 @@ def test_swa_slots_resolve_once_per_forward(manager_requests, monkeypatch): if not kernel.dsl_available(): pytest.skip("Counting native resolutions requires the CuTe DSL") manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) launches = [] refresh = kernel.refresh_swa_slots monkeypatch.setattr(kernel, "refresh_swa_slots", lambda *a: launches.append(refresh(*a))) layers = [manager.layout.layer(i) for i in range(len(manager.layout.compress_ratios))] def forward(): - metadata.begin_model_forward() + metadata.reset_forward_state() for layer in layers: metadata.enter_layer(layer) _prepare(metadata, requests, [0, 0], [257, 257]) forward() forward() - metadata.on_update_kv_lens() + metadata._refresh_live_endpoints() metadata.prepare_indexer(1) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): @@ -250,14 +265,15 @@ def forward(): @torch.inference_mode() def test_manager_partial_compression_graph_and_strided_publication(manager_requests): from tensorrt_llm._torch.attention.backends.sparse.csa2.compressor import CSA2Compressor - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( pack_rows, store_rows, ) manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) metadata = metadata.create_cuda_graph_metadata(2) compressor = CSA2Compressor(32, 512, 2, 1e-6).cuda() compressor.wkv_gate.weight.normal_(std=0.1) @@ -337,7 +353,6 @@ def test_odd_prefix_reuse_compression_matches_fresh(): CSA2CacheRole, ) from tensorrt_llm._torch.attention.backends.sparse.csa2.compressor import CSA2Compressor - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType @@ -366,7 +381,9 @@ def test_odd_prefix_reuse_compression_matches_fresh(): dtype=DataType.BF16, vocab_size=8192, ) - metadata = CSA2TrtllmMetadata(max_num_requests=3, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=3, max_num_tokens=1024, kv_cache_manager=manager + ) compressor = CSA2Compressor(32, 512, 2, 1e-6).cuda() compressor.wkv_gate.weight.normal_(std=0.1) torch.manual_seed(671) @@ -437,7 +454,6 @@ def compress(request_id, start, values): def test_chain_rewind_preserves_compressor_and_swa(start, accepted_drafts): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager from tensorrt_llm._torch.attention.backends.sparse.csa2.compressor import CSA2Compressor - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( gather_rows, @@ -474,7 +490,9 @@ def test_chain_rewind_preserves_compressor_and_swa(start, accepted_drafts): sampling_config=SamplingConfig(), is_streaming=False, ) - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=512, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=512, kv_cache_manager=manager + ) metadata.is_spec_decoding_enabled = True compressor = CSA2Compressor(32, 512, 2, 1e-6).cuda() compressor.wkv_gate.weight.normal_(std=0.1) @@ -556,11 +574,12 @@ def compress(position, values, context=False): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_temporal_prior_identity_rewind_and_replay(manager_requests): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.metadata import KVCacheParams manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager + ) metadata.is_cuda_graph = True resets = [] metadata.register_indexer_reset(1, lambda: resets.append(True)) @@ -607,10 +626,10 @@ def prepare(ordered, positions): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_retained_metadata_buffers_and_workspace_accounting(manager_requests): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) for length in (1, 2, 3, 4): _prepare(metadata, requests, [0, 0], [length, length]) eager_keys = [key for key in metadata._csa2_buffers if not key[-1]] @@ -637,7 +656,9 @@ def test_retained_metadata_buffers_and_workspace_accounting(manager_requests): def test_pageable_metadata_uploads_feed_captured_consumer(manager_requests): dtype = torch.int64 manager, _ = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager + ) metadata.is_cuda_graph = True width, steps = 257, 64 stream = torch.cuda.Stream() @@ -686,13 +707,8 @@ def test_pageable_metadata_uploads_feed_captured_consumer(manager_requests): @skip_pre_blackwell @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() -def test_representable_draft_switch_restores_target_fields(manager_requests): +def test_draft_metadata_reads_the_draft_cache_and_leaves_the_target(manager_requests): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata - from tensorrt_llm._torch.speculative.interface import ( - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, - ) from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType from tensorrt_llm.llmapi.llm_args import KvCacheConfig @@ -718,11 +734,12 @@ def test_representable_draft_switch_restores_target_fields(manager_requests): assert draft.prepare_context(request) assert draft.resize_context(request, request.context_chunk_size) draft._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=2, max_num_tokens=1024, kv_cache_manager=target, draft_kv_cache_manager=draft, + one_model_draft=True, ) _prepare(metadata, requests, [0, 0], [1, 1]) _assert_swa_cpu_reference(metadata, target, requests, [0, 0], [1, 1], [0, 0]) @@ -730,44 +747,27 @@ def test_representable_draft_switch_restores_target_fields(manager_requests): layer: _swa(metadata, layer)[0].clone() for layer in range(len(target.layout.compress_ratios)) } - target_pages = dict(metadata._csa2_swa_page_tables) - target_page_values = {layer: pages.clone() for layer, pages in target_pages.items()} target_positions = metadata.csa2_positions target_state = metadata.get_compression_batch(1).kv_state - resets = [] - metadata.register_indexer_reset(1, lambda: resets.append("target")) - saved = prepare_attn_metadata_for_draft_replay(metadata, draft) - assert resets == ["target"] - metadata.register_indexer_reset(1, lambda: resets.append("draft")) - try: - assert metadata.kv_cache_manager is draft - for layer, pages in metadata._csa2_swa_page_tables.items(): - torch.testing.assert_close( - target_pages[layer], target_page_values[layer], atol=0, rtol=0 - ) - _assert_swa_cpu_reference(metadata, draft, requests, [0, 0], [1, 1], [0, 0]) - finally: - restore_attn_metadata_after_draft_replay(metadata, saved) + + assert metadata.draft.kv_cache_manager is draft + _assert_swa_cpu_reference(metadata.draft, draft, requests, [0, 0], [1, 1], [0, 0]) + assert metadata.kv_cache_manager is target _assert_swa_cpu_reference(metadata, target, requests, [0, 0], [1, 1], [0, 0]) for layer, expected in target_reads.items(): torch.testing.assert_close(_swa(metadata, layer)[0], expected, atol=0, rtol=0) assert metadata.csa2_positions is target_positions assert metadata.get_compression_batch(1).kv_state is target_state - assert resets == ["target", "target"] - # A later draft switch restores its own cached callback/prior state; - # contiguous identity must not suppress reset of shared emission state. - saved = prepare_attn_metadata_for_draft_replay(metadata, draft) - assert resets[-2:] == ["target", "draft"] - restore_attn_metadata_after_draft_replay(metadata, saved) - assert resets[-1] == "target" - # Legacy direct manager rebinding remains legal: metadata-owned backing - # is refreshed from the new manager rather than rejected or redirected. - metadata.draft_kv_cache_manager = None + # Direct manager rebinding remains legal: metadata-owned backing is + # refreshed from the new manager rather than rejected or redirected. + rebound = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=target + ) for current in (draft, target): - metadata.kv_cache_manager = current - _prepare(metadata, requests, [0, 0], [1, 1]) - _assert_swa_cpu_reference(metadata, current, requests, [0, 0], [1, 1], [0, 0]) + rebound.core().kv_cache_manager = current + _prepare(rebound, requests, [0, 0], [1, 1]) + _assert_swa_cpu_reference(rebound, current, requests, [0, 0], [1, 1], [0, 0]) finally: for request in requests: draft.free_resources(request) @@ -845,11 +845,13 @@ def test_batch_page_tables_match_per_request_converters(manager_requests): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_temporal_prefill_seeds_first_decode(manager_requests, monkeypatch): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata + from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.metadata import KVCacheParams manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager + ) _prepare(metadata, requests, [0, 0], [2, 2]) assert metadata.prepare_indexer_prior(1, 4).shape == (0, 4) selected = torch.arange(16, dtype=torch.int32, device="cuda").view(4, 4) @@ -862,7 +864,7 @@ def test_temporal_prefill_seeds_first_decode(manager_requests, monkeypatch): if torch.cuda.get_device_capability()[0] != 10: return # Native FP4 paged descriptor coverage is validated on SM100. manager.get_index_pages(1).zero_() - assert metadata.prepare_indexer(1) is metadata + assert metadata.prepare_indexer(1) is metadata.ext(CSA2Metadata) # Index pages are read in place through the owner's block table. assert metadata.csa2_indexer_k_cache.data_ptr() == manager.get_index_pages(1).data_ptr() valid = metadata.csa2_indexer_valid_positions @@ -876,13 +878,14 @@ def test_temporal_prefill_seeds_first_decode(manager_requests, monkeypatch): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_graph_workspace_is_independent_of_context_gather_cap(manager_requests): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.metadata import KVCacheParams if torch.cuda.get_device_capability()[0] != 10: pytest.skip("Native FP4 paged descriptors require SM100-family GPUs") manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=8, kv_cache_manager=manager + ) metadata.is_cuda_graph = True metadata.request_ids = [request.py_request_id for request in requests] metadata.seq_lens = torch.ones(2, dtype=torch.int32) @@ -936,7 +939,9 @@ def replay_metadata(): assert manager._resume_and_restore(request_id, cache) assert cache.resize(32) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=64, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=64, kv_cache_manager=manager + ) yield manager, metadata for request_id in list(manager.kv_cache_map): manager.free_resources(SimpleNamespace(py_request_id=request_id)) @@ -1056,7 +1061,7 @@ def test_replay_requires_first_window_row_to_be_physically_writable(): # requires127..254. Page0 is genuinely absent in the real allocator. assert cache.resize(255, 255) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=128, kv_cache_manager=manager ) assert manager.get_swa_replay_ranges([255]) == ((127, 255),) @@ -1072,13 +1077,11 @@ def test_replay_requires_first_window_row_to_be_physically_writable(): @pytest.mark.parametrize("ratio", [1, 2]) @pytest.mark.parametrize("accepted_end", [128, 129, 130]) def test_live_endpoint_refreshes_query_slots_and_compressed_groups( - monkeypatch, ratio, accepted_end, query_width + ratio, accepted_end, query_width ): # Base scheduler invalidation is orthogonal; exercise the actual CSA2 tensor # refresh without native allocations, including an odd ratio-two boundary. - monkeypatch.setattr(TrtllmAttentionMetadata, "on_update_kv_lens", lambda self: None) - metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) - metadata._csa2_ready_for_kv_update = True + metadata = _bare_csa2_metadata() metadata.kv_cache_manager = SimpleNamespace( tokens_per_block=128, layout=SimpleNamespace(window_size=128, compress_ratios=(ratio,)) ) @@ -1112,8 +1115,11 @@ def test_live_endpoint_refreshes_query_slots_and_compressed_groups( query_width, ) } - identities = (metadata.csa2_positions.data_ptr(), metadata.csa2_main_write_slots[0].data_ptr()) - metadata.on_update_kv_lens() + identities = ( + metadata.csa2_positions.data_ptr(), + metadata.csa2_main_write_slots[0].data_ptr(), + ) + metadata._refresh_live_endpoints() positions = list(range(accepted_end - query_width, accepted_end)) assert metadata.csa2_positions.tolist() == positions # The layer resolves its SWA slots from the refreshed positions when entered. @@ -1160,6 +1166,9 @@ def test_routing_reset_detaches_shallow_clone(): indices = torch.tensor([[0]]) metadata.csa2_indices[0] = indices clone = copy(metadata) + routing = copy(metadata.ext(CSA2Metadata)) + routing.metadata = clone + clone.replace_ext(routing) clone.reset_routing() assert clone.csa2_indices == {} and clone.csa2_candidates == {} assert metadata.csa2_indices[0] is indices @@ -1173,13 +1182,13 @@ def test_routing_reset_detaches_shallow_clone(): @skip_pre_blackwell @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") def test_staging_excludes_out_of_capacity_rows() -> None: - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata + from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2BackendForwardArgs if torch.cuda.get_device_capability()[0] != 10: pytest.skip("Fused CSA2 gather requires SM100-family validation") q = torch.empty((2, 32, 512), dtype=torch.bfloat16, device="cuda") - metadata = CSA2TrtllmMetadata.for_query_tile(q, 17) + metadata = CSA2Metadata.for_query_tile(q, 17) swa = pack_rows(torch.randn(3, 512, dtype=torch.bfloat16, device="cuda"), "swa") main = pack_rows(torch.randn(3, 512, dtype=torch.bfloat16, device="cuda"), "main") swa_slots = torch.tensor([[0, 3, 2**40, -1], [3, -1, -1, -1]], device="cuda") @@ -1223,21 +1232,19 @@ def _assert_staging_bf16_decode(actual, expected): class _LifecycleMetadata(_FakeMetadata): - _get_csa2_buffer = CSA2TrtllmMetadata._get_csa2_buffer - _copy_host = CSA2TrtllmMetadata._copy_host - _copy_csa2_tensor = CSA2TrtllmMetadata._copy_csa2_tensor + _get_csa2_buffer = CSA2Metadata._get_csa2_buffer + _copy_host = CSA2Metadata._copy_host + _copy_csa2_tensor = CSA2Metadata._copy_csa2_tensor @property def num_generations(self) -> int: return self.num_seqs - self.num_contexts - def prepare(self): - super().prepare() + def prepare(self, inputs=None): + super().prepare(inputs) self._csa2_last_layer = -1 self.csa2_precomputed_kv_layers = set() - self.kv_lens_cuda[: self.num_seqs].copy_( - self.seq_lens + torch.tensor(self.kv_cache_params.num_cached_tokens_per_seq) - ) + self.kv_lens_cuda[: self.num_seqs].copy_(self.seq_lens + self.live_cached_lengths()) @pytest.mark.parametrize( @@ -1261,8 +1268,10 @@ def test_replay_preserves_live_speculative_kv_lifecycle( original_params = metadata.kv_cache_params metadata.prompt_lens = [400, 7, 700] metadata.kv_lens_cuda = torch.cat((metadata.kv_lens_cuda, torch.tensor([-99, -99]))) - metadata.kv_lens_cuda[2] -= 3 - metadata.on_update_kv_lens() + # The step's live speculative endpoint: the overlap correction the + # target's prepare applied. + metadata.kv_lens_offsets = torch.tensor([0, 0, -3], dtype=torch.int32) + metadata.prepare() expected_kv = metadata.kv_lens_cuda.clone() positions = metadata.csa2_positions.clone() untrimmed_swa = _swa(metadata, 0)[0][400:].clone() @@ -1278,7 +1287,7 @@ def test_replay_preserves_live_speculative_kv_lifecycle( assert plan.swa_floors == [400, 0, 0] torch.testing.assert_close(plan.rows, torch.arange(272, 413), atol=0, rtol=0) - # Encoder handoffs are published after planning and cleared by prepare(). + # Encoder handoffs are published after planning. topk = torch.arange(413 * 3, dtype=torch.int32).reshape(413, 3) candidates = torch.arange(413 * 2, dtype=torch.int32).reshape(413, 2) metadata.csa2_indices[20] = topk @@ -1286,27 +1295,35 @@ def test_replay_preserves_live_speculative_kv_lifecycle( metadata.csa2_precomputed_kv_layers = {20} metadata._csa2_last_layer = 20 saved_prompt = metadata.prompt_lens - original_buffer = metadata.kv_lens_cuda - enter_decoder_replay(metadata, plan) - assert metadata.prepare_calls == 1 - assert metadata.kv_lens_cuda is original_buffer - torch.testing.assert_close(metadata.kv_lens_cuda, expected_kv, atol=0, rtol=0) - torch.testing.assert_close(metadata.csa2_positions, positions[plan.rows], atol=0, rtol=0) - torch.testing.assert_close(metadata.csa2_indices[20], topk[plan.rows], atol=0, rtol=0) - torch.testing.assert_close(metadata.csa2_candidates[20], candidates[plan.rows], atol=0, rtol=0) - assert metadata.csa2_precomputed_kv_layers == {20} - assert metadata._csa2_last_layer == 20 - assert metadata.csa2_replay_query_rows is plan.rows - indices = _swa(metadata, 0)[0] + saved_seq_lens = metadata.seq_lens + prepare_calls = metadata.prepare_calls + replay = enter_decoder_replay(metadata, plan) + # The decoder layers run on the replay metadata; the target is not prepared again. + assert replay is metadata.decoder_replay_metadata + assert replay.prepare_calls == 1 + assert metadata.prepare_calls == prepare_calls + torch.testing.assert_close(replay.kv_lens_cuda[:3], expected_kv[:3], atol=0, rtol=0) + torch.testing.assert_close(replay.csa2_positions, positions[plan.rows], atol=0, rtol=0) + torch.testing.assert_close(replay.csa2_indices[20], topk[plan.rows], atol=0, rtol=0) + torch.testing.assert_close(replay.csa2_candidates[20], candidates[plan.rows], atol=0, rtol=0) + assert replay.csa2_precomputed_kv_layers == {20} + assert replay._csa2_last_layer == 20 + assert replay.csa2_replay_query_rows is plan.rows + indices = _swa(replay, 0)[0] torch.testing.assert_close((indices[:WINDOW] >= 0).sum(-1), torch.arange(1, WINDOW + 1)) torch.testing.assert_close(indices[WINDOW - 1], torch.arange(400, 528)) torch.testing.assert_close(indices[WINDOW:], untrimmed_swa, atol=0, rtol=0) - assert metadata.prompt_lens == [WINDOW, 7, 700] + assert replay.prompt_lens == [WINDOW, 7, 700] + # The target keeps the encoder pass's batch throughout. + assert metadata.seq_lens is saved_seq_lens + assert metadata.prompt_lens is saved_prompt + assert metadata.kv_cache_params is original_params + assert metadata.kv_cache_params.num_cached_tokens_per_seq == [128, 700, 900] + torch.testing.assert_close(metadata.kv_lens_cuda, expected_kv, atol=0, rtol=0) exit_decoder_replay(metadata, plan) assert metadata.request_ids == [10, 11, 12] and metadata.num_contexts == 2 assert metadata.kv_cache_params is original_params assert metadata.prompt_lens is saved_prompt - assert metadata.seq_lens is plan.saved_seq_lens assert metadata.seq_lens.tolist() == [400, 7, 6] assert metadata.kv_cache_params.num_cached_tokens_per_seq == [128, 700, 900] assert metadata.csa2_replay_query_rows is None @@ -1375,9 +1392,9 @@ def _assert_swa_cpu_reference(metadata, manager, requests, starts, lengths, floo def test_coalesced_uploads_mixed_dtypes_empty_values_and_slab_growth(): from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import _CoalescedUploads - metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) - vars(metadata).update( - is_cuda_graph=False, kv_lens_cuda=torch.empty(0, dtype=torch.int32, device="cuda") + metadata = _bare_csa2_metadata() + _assign( + metadata, is_cuda_graph=False, kv_lens_cuda=torch.empty(0, dtype=torch.int32, device="cuda") ) for size in (3, 1000): # The second round regrows the upload slab. values = { @@ -1386,7 +1403,7 @@ def test_coalesced_uploads_mixed_dtypes_empty_values_and_slab_growth(): "empty": np.empty(0, dtype=np.int64), "scalar": torch.tensor(size, dtype=torch.int32), } - uploads = _CoalescedUploads(metadata) + uploads = _CoalescedUploads(metadata.ext(CSA2Metadata)) outputs = {key: uploads.copy(key, value) for key, value in values.items()} uploads.flush() for key, value in values.items(): @@ -1397,10 +1414,10 @@ def test_coalesced_uploads_mixed_dtypes_empty_values_and_slab_growth(): @pytest.mark.cpu_only def test_swa_write_interval_validation_preserves_errors(): def check(pages, starts, lengths, floors, allocated, block): - rows, columns = CSA2TrtllmMetadata._swa_write_pages( + rows, columns = CSA2Metadata._swa_write_pages( starts, lengths, floors, allocated, block, pages.shape ) - CSA2TrtllmMetadata._check_written_pages(pages.numpy()[rows, columns] >= 0) + CSA2Metadata._check_written_pages(pages.numpy()[rows, columns] >= 0) full = [1 << 30] check(torch.tensor([[-1, 7]]), [128], [2], [0], full, 128) # Historical-only hole. @@ -1455,7 +1472,7 @@ def test_forward_swa_resolution_matches_cpu_reference(queries): assert manager.prepare_context(request) assert manager.resize_context(request, request.context_chunk_size) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=8192, kv_cache_manager=manager ) metadata.request_ids = [request.py_request_id] @@ -1465,7 +1482,6 @@ def test_forward_swa_resolution_matches_cpu_reference(queries): metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]) metadata.prepare() _assert_swa_cpu_reference(metadata, manager, [request], [0], [queries], [0]) - assert metadata._csa2_ready_for_kv_update finally: manager.free_resources(request) manager.shutdown() @@ -1495,15 +1511,18 @@ def test_swa_resolution_integer_outputs_and_graph(monkeypatch, native): pages_host = [row[:table_width] for row in pages_host] positions = torch.tensor(positions_host, dtype=torch.int32, device="cuda") requests = torch.tensor(requests_host, dtype=torch.int64, device="cuda") - metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) - vars(metadata).update( + metadata = _bare_csa2_metadata() + _assign( + metadata, is_cuda_graph=False, - _csa2_buffers={}, kv_lens_cuda=positions, kv_cache_manager=SimpleNamespace( tokens_per_block=block, layout=SimpleNamespace(window_size=window, compress_ratios=ratios), ), + ) + vars(metadata.ext(CSA2Metadata)).update( + _csa2_buffers={}, csa2_positions=positions, csa2_token_requests=requests, csa2_replay_start_positions=torch.tensor(floors_host, dtype=torch.int64, device="cuda"), @@ -1537,7 +1556,7 @@ def resolve_all(force=True): for name, out in zip( ("csa2_swa_indices", "csa2_swa_write_slots", "csa2_visible_lengths"), kept ): - out.copy_(getattr(metadata, name)[layer]) + out.copy_(getattr(metadata.ext(CSA2Metadata), name)[layer]) def check(): # Independent scalar CPU oracle, including invalid request domains. @@ -1651,13 +1670,13 @@ def test_indexer_descriptors_expand_pages_and_mask_missing_pages(): expected_valid, ) with torch.inference_mode(): - CSA2TrtllmMetadata._fill_indexer_descriptors( + CSA2Metadata._fill_indexer_descriptors( table, requests, decode_visible, 1, factor, 64, *outputs ) if device == "cuda" and page_size == 128 and columns: graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): - CSA2TrtllmMetadata._fill_indexer_descriptors( + CSA2Metadata._fill_indexer_descriptors( table, requests, decode_visible, 1, factor, 64, *outputs ) for value in outputs: @@ -1675,7 +1694,7 @@ def test_indexer_descriptors_expand_pages_and_mask_missing_pages(): @pytest.mark.cpu_only @pytest.mark.parametrize("published_first", [False, True]) def test_swa_table_backing_stays_fixed_across_publication_and_fallback(published_first): - metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) + metadata = _bare_csa2_metadata() metadata._csa2_buffers = {} metadata.kv_lens_cuda = torch.empty(0, dtype=torch.int32) metadata.is_cuda_graph = True @@ -1726,7 +1745,9 @@ def test_swa_fresh_publication_reuses_tables_and_keeps_captured_backing( from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheRole manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) metadata.is_cuda_graph = True # Physical publication order need not equal logical model-layer order. order = [2, 0, 3, 1] @@ -1769,7 +1790,7 @@ def remap(shift): kept = {layer: tuple(value.clone() for value in _swa(metadata, layer)) for layer in range(4)} def refresh_and_resolve(): - metadata.on_update_kv_lens() + metadata._refresh_live_endpoints() for layer, outputs in kept.items(): for value, out in zip(_swa(metadata, layer), outputs): out.copy_(value) @@ -1834,15 +1855,18 @@ def test_swa_layer_reference_matches_scalar_oracle(rows, window): if rows > 256: # Request 1 starts at floor 127: page 0 is history below the floor, never a write target. tables[1][1, 0] = -1 - metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) - vars(metadata).update( + metadata = _bare_csa2_metadata() + _assign( + metadata, is_cuda_graph=False, - _csa2_buffers={}, kv_lens_cuda=torch.empty(0, dtype=torch.int32), kv_cache_manager=SimpleNamespace( tokens_per_block=block, layout=SimpleNamespace(window_size=window, compress_ratios=ratios), ), + ) + vars(metadata.ext(CSA2Metadata)).update( + _csa2_buffers={}, csa2_positions=torch.tensor(positions, dtype=torch.int32), csa2_token_requests=requests_host, csa2_replay_start_positions=torch.tensor(floors, dtype=torch.int64), @@ -1905,7 +1929,7 @@ def test_swa_refresh_preserves_per_layer_page_domains(device: str) -> None: ) for pages, shape, ratio in zip(pages_host, shapes, ratios) ) - CSA2TrtllmMetadata._refresh_swa_tensor_outputs( + CSA2Metadata._refresh_swa_tensor_outputs( positions, requests, floors, block, window, layer_inputs ) for (_, reads, writes, visible, ratio), pages, (table_rows, width) in zip( @@ -1938,7 +1962,7 @@ def test_native_staging_pools_cover_tma_fetch_window( dtype: torch.dtype, shared_rows: int | None ) -> None: q = torch.empty(1, 8, 512, device="cuda", dtype=torch.bfloat16) - metadata = CSA2TrtllmMetadata.for_query_tile( + metadata = CSA2Metadata.for_query_tile( q, 0, staging_dtype=dtype, @@ -1965,7 +1989,7 @@ def test_native_staging_pools_cover_tma_fetch_window( @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_context_tile_cache_retains_latest_and_preserves_graph_and_dtype(): - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=64) + source = CSA2TrtllmAttention.Metadata(max_num_requests=2, max_num_tokens=64) q = torch.zeros(64, 8, 512, dtype=torch.bfloat16, device="cuda") generation = source.get_query_tile_metadata(q[:2], 0) generation.swa_pool.zero_() @@ -2042,7 +2066,9 @@ def test_context_tile_cache_retains_latest_and_preserves_graph_and_dtype(): @torch.inference_mode() def test_shared_context_domains_follow_owner_source_batch_and_replay(manager_requests): manager, requests = manager_requests - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + source = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) source.set_source_batch([200, 200], [0, 0]) _prepare(source, requests, [130, 131], [2, 1]) q = torch.zeros(3, 8, 512, device="cuda", dtype=torch.bfloat16) @@ -2105,11 +2131,13 @@ def test_shared_context_covers_upward_live_endpoint_inside_published_pages(manag from .test_backend import _backend manager, requests = manager_requests - source = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + source = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) source.set_source_batch([200, 200], [0, 0]) _prepare(source, requests, [130, 131], [2, 1]) source.kv_lens_cuda.fill_(260) - source.on_update_kv_lens() + source._refresh_live_endpoints() logical = torch.tensor([[110], [111], [112]], dtype=torch.int64, device="cuda") slots = source.global_slot_tile(1, 0, 3, logical) assert torch.all(slots >= 0) @@ -2125,7 +2153,7 @@ def test_shared_context_covers_upward_live_endpoint_inside_published_pages(manag shared = source.get_query_tile_metadata(q, 512, query_start=0, layer_idx=1) assert shared.shared_plan is not None assert shared.shared_plan.main_rows > 2 * (200 // 2) - selected = CSA2TrtllmMetadata.for_query_tile(q, 512, context_lengths=[2, 1]) + selected = CSA2Metadata.for_query_tile(q, 512, context_lengths=[2, 1]) backend = _backend(8) def arguments(): @@ -2156,7 +2184,9 @@ def test_compressor_readiness_uses_current_rows(manager_requests): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheRole manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) base = manager.host_kv_cache_block_offsets row = int(manager.index_mapper.get_copy_index([requests[0].py_request_id], 1, 1)[0]) @@ -2185,7 +2215,9 @@ def prepare_source(length): @torch.inference_mode() def test_first_decode_after_context_takes_full_prepare(manager_requests): manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) # A one-token context chunk (a full prefix hit) and then its first decode # step: the context geometry must not be reused for the generation rows. _prepare(metadata, requests, [126, 129], [1, 1]) @@ -2196,6 +2228,7 @@ def test_first_decode_after_context_takes_full_prepare(manager_requests): def _decode_derived_outputs(metadata): # SWA outputs are resolved when a forward enters its first layer. + csa2 = metadata.ext(CSA2Metadata) metadata._ensure_swa_slots() outputs = {"positions": metadata.csa2_positions} for name in ( @@ -2205,24 +2238,20 @@ def _decode_derived_outputs(metadata): "csa2_main_write_slots", "_csa2_compressed_positions", ): - outputs.update({(name, layer): tensor for layer, tensor in getattr(metadata, name).items()}) + outputs.update({(name, layer): tensor for layer, tensor in getattr(csa2, name).items()}) for owner, batch in metadata._csa2_compression.items(): for name in ("start_positions", "kv_lengths", "cu_compressed_lengths"): outputs[owner, name] = getattr(batch, name) return outputs -def _prepare_decode_metadata(metadata, requests, starts, *, deferred=False): - from contextlib import nullcontext - +def _prepare_decode_metadata(metadata, requests, starts): metadata.request_ids = [request.py_request_id for request in requests] metadata.seq_lens = torch.ones(len(requests), dtype=torch.int32) metadata.num_contexts = 0 metadata.prompt_lens = [request.prompt_len for request in requests] metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=starts) - with metadata.defer_cuda_graph_decode_outputs() if deferred else nullcontext(): - metadata.prepare() - assert not metadata._csa2_defer_decode_outputs + metadata.prepare() @skip_pre_blackwell @@ -2230,11 +2259,13 @@ def _prepare_decode_metadata(metadata, requests, starts, *, deferred=False): @torch.inference_mode() def test_steady_decode_refreshes_page_tables_across_pages(manager_requests, monkeypatch): manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) refreshed = [] refresh = metadata._refresh_steady_page_tables monkeypatch.setattr( - metadata, + metadata.ext(CSA2Metadata), "_refresh_steady_page_tables", lambda *args: refreshed.append(args[2][0]) or refresh(*args), ) @@ -2243,7 +2274,7 @@ def test_steady_decode_refreshes_page_tables_across_pages(manager_requests, monk for start in range(125, 131): starts = [start, start + 3] _prepare_decode_metadata(metadata, requests, starts) - reference = CSA2TrtllmMetadata( + reference = CSA2TrtllmAttention.Metadata( max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager ) _prepare_decode_metadata(reference, requests, starts) @@ -2259,19 +2290,21 @@ def test_steady_decode_refreshes_page_tables_across_pages(manager_requests, monk @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("batch_size", [1, 2]) @torch.inference_mode() -def test_decode_graph_defers_derived_uploads_and_refreshes_all_outputs( +def test_decode_graph_reads_the_outputs_each_prepare_writes( manager_requests, monkeypatch, batch_size ): + """A decode graph's consumers read what every eager prepare derived, in place.""" from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID - from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.inputs import DecoderInputPreparer from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm.bindings import SamplingConfig manager, requests = manager_requests - reference = CSA2TrtllmMetadata( + reference = CSA2TrtllmAttention.Metadata( + max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager + ) + base = CSA2TrtllmAttention.Metadata( max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager ) - base = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) metadata = base.create_cuda_graph_metadata(batch_size) copied = [] copy_host = metadata._copy_host @@ -2280,26 +2313,21 @@ def record_copy(key, destination, value): copied.append(key) return copy_host(key, destination, value) - monkeypatch.setattr(metadata, "_copy_host", record_copy) + monkeypatch.setattr(metadata.ext(CSA2Metadata), "_copy_host", record_copy) selected = requests[:batch_size] starts = [127] * batch_size - _prepare_decode_metadata(metadata, selected, starts, deferred=True) - assert metadata._csa2_deferred_decode_outputs - inputs = object.__new__(DecoderInputPreparer) - inputs.guided_decoder = None + _prepare_decode_metadata(metadata, selected, starts) pointers = { name: tensor.data_ptr() for name, tensor in _decode_derived_outputs(metadata).items() } def forward(): - # The real engine's preprocessing is captured ahead of every consumer. - inputs.preprocess_inputs( - {"attn_metadata": metadata}, enable_spec_decode=False, runtime_draft_len=0 - ) return {name: tensor.clone() for name, tensor in _decode_derived_outputs(metadata).items()} + # As in the engine, every warmup forward and the capture follow a prepare. for _ in range(2): forward() + _prepare_decode_metadata(metadata, selected, starts) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): observed = forward() @@ -2310,7 +2338,7 @@ def forward(): for name, tensor in _decode_derived_outputs(reference).items(): torch.testing.assert_close(observed[name], tensor, atol=0, rtol=0) # Consecutive decode steps inside the same pages take the upstream steady - # preparation path. Selected graphs still own every live endpoint refresh. + # preparation path, which refreshes the live endpoints on device. for steady_start in (130, 131, 132): starts = [steady_start] * batch_size _prepare_decode_metadata(reference, selected, starts) @@ -2318,12 +2346,9 @@ def forward(): name: tensor.clone() for name, tensor in _decode_derived_outputs(reference).items() } copied.clear() - _prepare_decode_metadata(metadata, selected, starts, deferred=True) - assert metadata._csa2_deferred_decode_outputs + _prepare_decode_metadata(metadata, selected, starts) if steady_start > 130: assert not copied - for tensor in _decode_derived_outputs(metadata).values(): - tensor.fill_(-997) graph.replay() for name in expected: torch.testing.assert_close(observed[name], expected[name], atol=0, rtol=0) @@ -2368,12 +2393,10 @@ def forward(): expected = { name: tensor.clone() for name, tensor in _decode_derived_outputs(reference).items() } - _prepare_decode_metadata(metadata, selected, starts, deferred=True) + _prepare_decode_metadata(metadata, selected, starts) assert pointers == { name: tensor.data_ptr() for name, tensor in _decode_derived_outputs(metadata).items() } - for tensor in _decode_derived_outputs(metadata).values(): - tensor.fill_(-997) graph.replay() for name in expected: torch.testing.assert_close(observed[name], expected[name], atol=0, rtol=0) @@ -2381,54 +2404,23 @@ def forward(): metadata, manager, selected, starts, [1] * batch_size, [0] * batch_size ) - # Graph metadata used by a direct caller must still be fully initialized. - _prepare_decode_metadata(metadata, selected, starts) - assert not metadata._csa2_deferred_decode_outputs - for name, tensor in _decode_derived_outputs(metadata).items(): - torch.testing.assert_close(tensor, expected[name], atol=0, rtol=0) +def test_bind_remote_tail_reads_context_requests(): + """The remote-tail role and starts come from the bound context requests.""" -@skip_pre_blackwell -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -@pytest.mark.parametrize( - "fallback,manager_requests", - [ - ("eager", {}), - ("context", {}), - ("source", {}), - ("remote_tail", {}), - ("speculative", {"speculative": True}), - ], - indirect=["manager_requests"], -) -@torch.inference_mode() -def test_decode_graph_deferral_keeps_unsupported_prepare_paths_complete(manager_requests, fallback): - manager, requests = manager_requests - metadata = CSA2TrtllmMetadata(max_num_requests=2, max_num_tokens=1024, kv_cache_manager=manager) - metadata.is_cuda_graph = fallback != "eager" - metadata.request_ids = [request.py_request_id for request in requests] - metadata.seq_lens = torch.ones(2, dtype=torch.int32) - metadata.num_contexts = 2 if fallback == "context" else 0 - metadata.prompt_lens = [257, 257] - metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[127, 128]) - if fallback == "source": - metadata.set_source_batch([1, 1], [127, 128]) - if fallback == "remote_tail": - metadata.csa2_remote_tail_mode = "decode" - if fallback == "speculative": - metadata.is_spec_decoding_enabled = True - with metadata.defer_cuda_graph_decode_outputs(): - metadata.prepare() - assert not metadata._csa2_deferred_decode_outputs - assert metadata.csa2_positions.tolist() == [127, 128] - _assert_swa_cpu_reference(metadata, manager, requests, [127, 128], [1, 1], [0, 0]) + def request(mode, start): + return SimpleNamespace(py_csa2_remote_tail_mode=mode, py_csa2_remote_tail_start=start) + csa2 = SimpleNamespace() + CSA2Metadata._bind_remote_tail(csa2, [request("destination", 64), request("destination", 8)]) + assert csa2.csa2_remote_tail_mode == "destination" + assert csa2.csa2_remote_tail_starts == (64, 8) -@pytest.mark.cpu_only -def test_decode_graph_prepare_scope_restores_proof_after_exception(): - metadata = object.__new__(CSA2TrtllmMetadata) - with pytest.raises(RuntimeError, match="prepare failed"): - with metadata.defer_cuda_graph_decode_outputs(): - assert metadata._csa2_defer_decode_outputs - raise RuntimeError("prepare failed") - assert not metadata._csa2_defer_decode_outputs + CSA2Metadata._bind_remote_tail(csa2, [request(None, None)]) + assert csa2.csa2_remote_tail_mode is None + assert csa2.csa2_remote_tail_starts == () + + with pytest.raises(ValueError, match="homogeneous"): + CSA2Metadata._bind_remote_tail(csa2, [request("source", 1), request(None, None)]) + with pytest.raises(ValueError, match="homogeneous"): + CSA2Metadata._bind_remote_tail(csa2, [request("source", 1), request("destination", 2)]) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_metadata_uploads.py b/tests/unittest/_torch/attention/sparse/csa2/test_metadata_uploads.py index 4345bc477935..cfea10a20c56 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_metadata_uploads.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_metadata_uploads.py @@ -8,8 +8,11 @@ import pytest import torch +from _torch.helpers import bare_extension -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata skip_pre_blackwell = pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.get_device_capability() < (10, 0), @@ -28,15 +31,25 @@ ] +def _bare_csa2_metadata() -> TrtllmAttentionMetadata: + """Metadata built without ``__init__``, carrying a bare CSA2 extension.""" + metadata = object.__new__(TrtllmAttentionMetadata) + metadata.replace_ext(bare_extension(CSA2Metadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) + return metadata + + def _metadata(device): - metadata = object.__new__(CSA2TrtllmMetadata) + metadata = _bare_csa2_metadata() metadata.is_cuda_graph = False metadata.num_sparse_topk = 256 - for name in ("query_lens_host", "kv_lens", "prompt_lens_cpu", "host_request_types"): - setattr(metadata, name, torch.empty(4, dtype=torch.int32, device="cpu")) - for name in ("query_lens_device", "kv_lens_cuda", "prompt_lens_cuda"): - setattr(metadata, name, torch.empty(4, dtype=torch.int32, device=device)) + for name in ("kv_lens", "prompt_lens_cpu", "host_request_types"): + setattr(metadata.core(), name, torch.empty(4, dtype=torch.int32, device="cpu")) + for name in ("kv_lens_cuda", "prompt_lens_cuda"): + setattr(metadata.core(), name, torch.empty(4, dtype=torch.int32, device=device)) metadata.host_total_kv_lens = torch.zeros(2, dtype=torch.int32, device="cpu") + metadata.query_lens_host = torch.empty(4, dtype=torch.int32, device="cpu") + metadata.query_lens_device = torch.empty(4, dtype=torch.int32, device=device) metadata.prepared_cu_q = torch.empty(5, dtype=torch.int32, device=device) metadata.prepared_cu_kv = torch.empty(5, dtype=torch.int32, device=device) return metadata diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_module.py b/tests/unittest/_torch/attention/sparse/csa2/test_module.py index a6c2dcfd3e0b..ee7440e9a616 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_module.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_module.py @@ -12,6 +12,7 @@ from torch import nn from utils.util import skip_blackwell_geforce +from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager from tensorrt_llm._torch.attention.backends.sparse.csa2.params import ( CSA2Layout, @@ -107,17 +108,21 @@ def write_layer_rows(layer, slots, values, transform=None, *, global_rows=None, select_global_source=lambda layer, local, global_hidden: local, enter_layer=lambda layer: entered.append(layer.layer_idx), ) + # The stubs serve as both the core metadata and its CSA2 extension. + source.ext = source.find_ext = lambda cls: source def phase_metadata(query, extra_width, query_start, *, staging_dtype, layer_idx=None): stop = query_start + query.shape[0] assert stop <= (contexts if query_start < contexts else count) torch.testing.assert_close(query.flatten(1), hidden[query_start:stop], atol=0, rtol=0) - return SimpleNamespace( + tile = SimpleNamespace( num_contexts=int(query_start < contexts), num_tokens=query.shape[0], num_query_heads=1, swa_pool=torch.empty(0, dtype=staging_dtype), ) + tile.ext = lambda cls: tile + return tile def map_indices(layer, start, end, logical, *, visible_lengths): torch.testing.assert_close(logical[:, 0], row_ids[start:end], atol=0, rtol=0) @@ -271,20 +276,21 @@ def test_ced_owner_is_written_once_before_narrowed_query(monkeypatch, module_cac PositionalEmbeddingParams, RopeParams, ) - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.module import DeepseekV41Attention from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm.functional import PositionEmbeddingType manager, request = module_cache - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager + ) metadata.request_ids = [request.py_request_id] metadata.num_contexts = 1 metadata.prompt_lens = [4] metadata.seq_lens = torch.tensor([4], dtype=torch.int32) metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]) metadata.prepare() - metadata.begin_model_forward() + metadata.reset_forward_state() torch.manual_seed(4126) with torch.device("cuda"): model = DeepseekV41Attention( @@ -345,7 +351,6 @@ def _check_ratio2_module_partial_groups(monkeypatch, module_cache, overlap, impl PositionalEmbeddingParams, RopeParams, ) - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.module import DeepseekV41Attention from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import ( pack_rows, @@ -361,7 +366,9 @@ def _check_ratio2_module_partial_groups(monkeypatch, module_cache, overlap, impl torch.manual_seed(818) manager, request = module_cache layout = manager.layout - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager + ) metadata.request_ids = [request.py_request_id] metadata.num_contexts = 1 metadata.prompt_lens = [4] @@ -783,7 +790,6 @@ def test_fused_native_index_q_and_graph(monkeypatch): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @torch.inference_mode() def test_native_module_auxiliary_stream_and_graph(monkeypatch, module_cache): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.metadata import KVCacheParams torch.manual_seed(946) @@ -795,7 +801,9 @@ def test_native_module_auxiliary_stream_and_graph(monkeypatch, module_cache): "tensorrt_llm._torch.attention.backends.sparse.csa2.module.do_multi_stream", lambda: enabled[0], ) - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager + ) metadata.request_ids = [request.py_request_id] metadata.num_contexts = 1 metadata.prompt_lens = [4] @@ -841,7 +849,6 @@ def run(): @pytest.mark.parametrize("overlap", [False, True]) @torch.inference_mode() def test_native_module_fixed_context_graph(monkeypatch, module_cache, overlap): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.metadata import KVCacheParams torch.manual_seed(947) @@ -852,7 +859,9 @@ def test_native_module_fixed_context_graph(monkeypatch, module_cache, overlap): monkeypatch.setattr( "tensorrt_llm._torch.attention.backends.sparse.csa2.module.do_multi_stream", lambda: overlap ) - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager + ) metadata.request_ids = [request.py_request_id] metadata.num_contexts = 1 metadata.prompt_lens = [4] @@ -960,7 +969,6 @@ def test_packed_output_inverse_rope_module_and_graph( from dataclasses import replace - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.metadata import KVCacheParams torch.manual_seed(950) @@ -972,7 +980,9 @@ def test_packed_output_inverse_rope_module_and_graph( grouped_block=block, sparse_options={"use_packed_sparse_attention": True, "fuse_packed_output_rope": True}, ) - metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager + ) metadata.request_ids = [request.py_request_id] metadata.num_contexts = 1 metadata.prompt_lens = [4] @@ -1034,7 +1044,6 @@ def run(): @torch.inference_mode() def test_two_rank_module_projection_parity(monkeypatch, quantization): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.nccl_window_tensor_scope import nccl_window_tensor_scope @@ -1111,7 +1120,7 @@ def fresh_metadata(mapping): assert manager.prepare_context(request) assert manager.resize_context(request, request.context_chunk_size) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=3, kv_cache_manager=manager ) metadata.request_ids = [request.py_request_id] @@ -1150,7 +1159,6 @@ def replay_module_factory(): RopeParams, ) from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata from tensorrt_llm._torch.attention.backends.sparse.csa2.module import DeepseekV41Attention from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest @@ -1194,7 +1202,7 @@ def create(ratio, input_length=8): assert manager.prepare_context(request) assert manager.resize_context(request, request.context_chunk_size) manager._stream.synchronize() - metadata = CSA2TrtllmMetadata( + metadata = CSA2TrtllmAttention.Metadata( max_num_requests=1, max_num_tokens=16, kv_cache_manager=manager ) metadata.request_ids = [request.py_request_id] @@ -1440,13 +1448,14 @@ def observe(x): @torch.inference_mode() def test_bounded_replay_full_module_graph_changes_hit_position(replay_module_factory): from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheRole - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata torch.manual_seed(957) model, manager, seed_metadata, hidden = replay_module_factory(2, input_length=12) def new_metadata(graph=False): - result = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=16, kv_cache_manager=manager) + result = CSA2TrtllmAttention.Metadata( + max_num_requests=1, max_num_tokens=16, kv_cache_manager=manager + ) result.request_ids = list(seed_metadata.request_ids) result.num_contexts = 1 result.prompt_lens = [12] diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py index 9b63fe8ada4a..f81ee369677a 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py @@ -26,6 +26,7 @@ import torch._dynamo import torch.nn as nn import torch.nn.functional as F +from _torch.helpers import bare_extension from tensorrt_llm._torch.attention.backends.interface import ( MLAParams, @@ -37,7 +38,7 @@ from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import ( DeepseekV4CacheManager, DeepseekV4Indexer, - DeepseekV4TrtllmAttentionMetadata, + DeepseekV4Metadata, ) from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.compressor import ( Compressor, @@ -47,6 +48,8 @@ DEEPSEEK_V4_SLIDING_ATTENTION, DeepseekV4AttentionType, ) +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._utils import get_sm_version @@ -139,6 +142,10 @@ def __init__( self.kv_lens_cuda_runtime = None # Set by caller self.cached_token_lens_cuda = None # Set by caller + def ext(self, cls: type) -> "DummyAttentionMetadata": + """The dummy carries the DeepSeek-V4 extension state itself.""" + return self + # ============================================================================ # Reference Implementation (DO NOT MODIFY) @@ -371,6 +378,16 @@ def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor): ROPE_THETA, ROPE_FACTOR, BETA_FAST, BETA_SLOW = 40000.0, 4, 32, 1 +def _dsv4_stub() -> DeepseekV4Metadata: + """An extension built through ``__new__`` on a core metadata built the same way.""" + metadata = object.__new__(TrtllmAttentionMetadata) + extension = object.__new__(DeepseekV4Metadata) + extension.metadata = metadata + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + metadata.replace_ext(extension) + return extension + + def _active_compressed_position_ids( compress_ratio, num_contexts, num_gen_tokens_per_seq, cached_tokens, kv_lens ): @@ -380,9 +397,10 @@ def _active_compressed_position_ids( cached_tokens = torch.tensor(cached_tokens, dtype=torch.int32, device=DEVICE) kv_lens = torch.tensor(kv_lens, dtype=torch.int32, device=DEVICE) - metadata = object.__new__(DeepseekV4TrtllmAttentionMetadata) - metadata.num_contexts = num_contexts - metadata.num_generations = num_generations + metadata = _dsv4_stub() + md = metadata.metadata + md.num_contexts = num_contexts + md.num_generations = num_generations metadata.num_gen_tokens_per_seq = num_gen_tokens_per_seq metadata._compress_ratios_sorted = [compress_ratio] metadata.compressed_kv_lens_cuda = { @@ -496,7 +514,7 @@ def run(new_comp_by_ratio): new_comp_bufs[r][:batch_size] = counts cu_bufs[r][: batch_size + 1] = F.pad(torch.cumsum(counts, 0), (1, 0)).to(torch.int32) batch_size_cuda.fill_(batch_size) - DeepseekV4TrtllmAttentionMetadata._compute_compressed_mask( + DeepseekV4Metadata._compute_compressed_mask( new_comp_bufs, cu_bufs, mask_bufs, batch_size_cuda, ratios ) for r, new_comp in new_comp_by_ratio.items(): @@ -570,7 +588,7 @@ def run( gen_params[0].fill_(num_contexts) gen_params[1].fill_(num_generations) gen_params[2].fill_(num_gen_tokens_per_seq) - DeepseekV4TrtllmAttentionMetadata._compute_gen_compressed_position_ids( + DeepseekV4Metadata._compute_gen_compressed_position_ids( past_bufs, cu_bufs, pos_bufs, gen_params, ratios ) for r in ratios: @@ -655,7 +673,7 @@ def test_ctx_position_ids_host_sizes_match_device_scalar_fallback( def _run(ctx_output_sizes): out = torch.full((total + 8,), -1, dtype=torch.int32, device=DEVICE) - DeepseekV4TrtllmAttentionMetadata._compute_ctx_compressed_position_ids( + DeepseekV4Metadata._compute_ctx_compressed_position_ids( {compress_ratio: past}, {compress_ratio: cu}, {compress_ratio: out}, diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_offload.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_offload.py index a10f0fb9ed74..dc4e7056d6a4 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_offload.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_offload.py @@ -20,7 +20,9 @@ import pytest import torch +from _torch.helpers import bare_extension +from tensorrt_llm._torch.attention.backends.features import DraftMetadata from tensorrt_llm._torch.attention.backends.interface import AttentionInputType from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import backend as deepseek_v4_backend from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import ( @@ -29,9 +31,7 @@ from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.kernels import ( deepseek_v4_local_to_global_indices, ) -from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( - DeepseekV4TrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import DeepseekV4Metadata from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.params import DeepseekV4AttentionType from tensorrt_llm._torch.attention.backends.sparse.offload import ( NVFP4_SCRATCH_TAIL_ROWS, @@ -39,6 +39,8 @@ _SparseBatchMember, allocate_sparse_offload_state, ) +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.memory_buffer_utils import Buffers from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import CUDA_GRAPH_DUMMY_REQUEST_ID from tensorrt_llm.bindings import DataType @@ -52,18 +54,23 @@ def _metadata(manager, *, enabled=True, graph=False, buffers=None, initialize=True): # Exercise the production offload initialization without constructing the # unrelated indexer/compressor/native attention workspaces. - metadata = object.__new__(DeepseekV4TrtllmAttentionMetadata) - metadata.kv_cache_manager = manager - metadata.sparse_metadata_params = DeepSeekV4SparseAttentionConfig( + core = object.__new__(TrtllmAttentionMetadata) + metadata = object.__new__(DeepseekV4Metadata) + metadata.metadata = core + core.replace_ext(bare_extension(MTPIndexShareExtension, core)) + core.replace_ext(metadata) + core.core().kv_cache_manager = manager + core.sparse_metadata_params = DeepSeekV4SparseAttentionConfig( enable_kv_cache_offload=enabled, index_topk=8 ).to_sparse_metadata_params() - metadata.max_num_sequences = 4 + core.core().max_num_sequences = 4 metadata.sparse_mla_topk = 8 - metadata.beam_width = 1 + metadata.mixed_precision_high_block_table = None + core.core().beam_width = 1 metadata.max_draft_tokens = 0 - metadata.draft_kv_cache_manager = None - metadata.is_cuda_graph = graph - metadata.cuda_graph_buffers = buffers + core.replace_ext(bare_extension(DraftMetadata, core)) + core.core().is_cuda_graph = graph + core.core().cuda_graph_buffers = buffers if initialize: metadata._init_sparse_offload_state() return metadata @@ -83,7 +90,7 @@ def test_sparse_offload_rejects_configuration_before_allocation( manager.pp_layers = [3] metadata = _metadata(manager, initialize=False) if unsupported == "beam": - metadata.beam_width = 2 + metadata.metadata.beam_width = 2 elif unsupported == "configured_beam": manager.max_beam_width = 2 elif unsupported == "draft": @@ -93,8 +100,8 @@ def test_sparse_offload_rejects_configuration_before_allocation( elif unsupported == "draft_tree": manager.max_total_draft_tokens = 1 else: - metadata.draft_kv_cache_manager = object() - with patch.object(metadata, "get_empty") as allocate: + metadata.metadata.draft_kv_cache_manager = object() + with patch.object(metadata.metadata, "get_empty") as allocate: with pytest.raises(NotImplementedError, match="single-beam, non-speculative"): metadata._init_sparse_offload_state() allocate.assert_not_called() @@ -122,9 +129,9 @@ def test_sparse_offload_stage_without_local_csa_allocates_nothing(): def test_sparse_metadata_uses_native_capacity_and_configured_width(): manager = _manager() metadata = _metadata(manager, initialize=False) - metadata.max_num_sequences = 2 + metadata.metadata.core().max_num_sequences = 2 with patch.object( - metadata, + metadata.metadata, "get_empty", side_effect=lambda buffers, shape, **kwargs: torch.empty(shape, dtype=torch.int32), ): @@ -172,7 +179,7 @@ def allocate(name, shape, dtype): def test_sparse_metadata_rejects_different_selection_width(): metadata = _metadata(_manager(), initialize=False) metadata.sparse_mla_topk = 4 - with patch.object(metadata, "get_empty") as allocate: + with patch.object(metadata.metadata, "get_empty") as allocate: with pytest.raises(ValueError, match="configured positive top-k"): metadata._init_sparse_offload_state() allocate.assert_not_called() @@ -203,7 +210,7 @@ def test_sparse_offload_buffers_are_persistent_and_separate(graph): assert len({t.data_ptr() for t in tensors}) == len(tensors) if graph: smaller = _metadata(manager, graph=True, buffers=buffers, initialize=False) - smaller.max_num_sequences = 2 + smaller.metadata.core().max_num_sequences = 2 smaller._init_sparse_offload_state() reused = smaller.sparse_offload_state assert reused.selected_history_units.data_ptr() == state.selected_history_units.data_ptr() @@ -339,9 +346,9 @@ def test_metadata_routes_only_sparse_main_table(enabled): manager._enable_kv_cache_offload = enabled manager._stream = torch.cuda.current_stream() metadata = _metadata(manager, enabled=enabled) - metadata.request_ids = [20, 10] - metadata._seq_lens = torch.ones(2, dtype=torch.int32) - metadata._num_contexts = 0 + metadata.metadata.request_ids = [20, 10] + metadata.metadata._seq_lens = torch.ones(2, dtype=torch.int32) + metadata.metadata._num_contexts = 0 metadata.sliding_block_tables = torch.empty((1, 1, 4, 6), dtype=torch.int32, device="cuda") metadata.compress_block_tables = { ratio: torch.empty((4, 6), dtype=torch.int32, device="cuda") for ratio in (4, 128) @@ -351,7 +358,7 @@ def test_metadata_routes_only_sparse_main_table(enabled): side_effect=lambda out, *args, **kwargs: out.fill_(29) ) if enabled: - _publish(manager, metadata.request_ids) + _publish(manager, metadata.metadata.request_ids) metadata.prepare_for_block_tables() assert (metadata.sliding_block_tables == 71).all() assert (metadata.compress_block_tables[128] == 29).all() @@ -770,12 +777,19 @@ def _host_batch(*, prefill=False): prompt_lens=[8, 12], kv_cache_params=SimpleNamespace(num_cached_tokens_per_seq=[0, 0]), ) + # The stub serves as both the core metadata and its extension. + metadata.metadata = metadata + metadata.index_share = metadata + # Not a draft: it takes no target's state. + metadata.target_source = lambda: None metadata._validate_ragged_verification = MethodType( - DeepseekV4TrtllmAttentionMetadata._validate_ragged_verification, metadata + DeepseekV4Metadata._validate_ragged_verification, metadata ) metadata._validate_dsa_sparse_offload_batch = MethodType( - DeepseekV4TrtllmAttentionMetadata._validate_dsa_sparse_offload_batch, metadata + DeepseekV4Metadata._validate_dsa_sparse_offload_batch, metadata ) + # The stub has no lazily built length prefixes to invalidate. + metadata._invalidate_length_prefixes = lambda: None return metadata @@ -787,9 +801,9 @@ def test_offload_admits_only_decode(prefill): metadata = _host_batch(prefill=prefill) if prefill: with pytest.raises(NotImplementedError, match="decode batches only"): - DeepseekV4TrtllmAttentionMetadata.validate_sparse_offload_batch(metadata) + DeepseekV4Metadata.validate_sparse_offload_batch(metadata) else: - DeepseekV4TrtllmAttentionMetadata.validate_sparse_offload_batch(metadata) + DeepseekV4Metadata.validate_sparse_offload_batch(metadata) assert not metadata.sparse_offload_state.prepared @@ -820,9 +834,9 @@ def test_offload_rejects_batch_before_table_preparation(unsupported): else: metadata.seq_lens_kv[0] += 1 metadata.validate_sparse_offload_batch = lambda: ( - DeepseekV4TrtllmAttentionMetadata.validate_sparse_offload_batch(metadata) + DeepseekV4Metadata.validate_sparse_offload_batch(metadata) ) with pytest.raises(NotImplementedError): - DeepseekV4TrtllmAttentionMetadata.prepare(metadata) + DeepseekV4Metadata.prepare(metadata) metadata.kv_cache_manager.compute_sliding_block_tables.assert_not_called() assert not metadata.sparse_offload_state.prepared diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py index 9d6cc8cf66b6..4da88c014553 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py @@ -23,8 +23,10 @@ import pytest import torch +from _torch.helpers import construct_extension, refresh_from_lengths from utils.util import skip_pre_blackwell +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionInputType, MLAParams, @@ -34,10 +36,11 @@ from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import ( DeepseekV4AttentionType, DeepseekV4CacheManager, + DeepseekV4Metadata, DeepseekV4TrtllmAttention, - DeepseekV4TrtllmAttentionMetadata, ) from tensorrt_llm._torch.attention.backends.sparse.params import SparseBackendForwardArgs +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests @@ -174,6 +177,17 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) +def _create_metadata( + *, sparse_attention_config, enable_flash_mla: bool = False, **kwargs +) -> TrtllmAttentionMetadata: + """DeepSeek-V4 attention metadata, built the way the backend builds it.""" + return DeepseekV4TrtllmAttention.Metadata( + sparse_metadata_params=sparse_attention_config.to_sparse_metadata_params(), + extension_types=(FlashMlaMetadata,) if enable_flash_mla else (), + **kwargs, + ) + + def _create_cache_manager(scenario: Scenario, context_lengths: List[int], max_seq_len: int): sparse_config = DeepSeekV4SparseAttentionConfig( index_n_heads=64, @@ -723,7 +737,7 @@ def test_deepseek_v4_sparse_mla_single_token_tp4_local_heads_repro(): attn_sink = torch.randn(local_num_heads, dtype=torch.float32, device=device).mul_(0.5) layer.attn_sink = torch.nn.Parameter(attn_sink, requires_grad=False) - attn_metadata = DeepseekV4TrtllmAttentionMetadata( + attn_metadata = _create_metadata( seq_lens=torch.tensor(context_lengths, dtype=torch.int), request_ids=[0], max_num_requests=1, @@ -1054,7 +1068,7 @@ def yarn_get_mscale(scale=1, mscale=1): # 7. Context phase ctx_seq_lens = torch.tensor(context_lengths, dtype=torch.int) - attn_metadata = DeepseekV4TrtllmAttentionMetadata( + attn_metadata = _create_metadata( seq_lens=ctx_seq_lens, request_ids=request_ids, max_num_requests=batch_size, @@ -1188,8 +1202,8 @@ def yarn_get_mscale(scale=1, mscale=1): gen_seq_lens = torch.tensor(generation_lens, dtype=torch.int) total_gen_tokens = sum(generation_lens) - def create_gen_metadata(ragged: bool) -> DeepseekV4TrtllmAttentionMetadata: - metadata = DeepseekV4TrtllmAttentionMetadata( + def create_gen_metadata(ragged: bool) -> TrtllmAttentionMetadata: + metadata = _create_metadata( seq_lens=gen_seq_lens, request_ids=request_ids, max_num_requests=batch_size, @@ -1204,8 +1218,11 @@ def create_gen_metadata(ragged: bool) -> DeepseekV4TrtllmAttentionMetadata: mapping=mapping, enable_flash_mla=torch.cuda.get_device_capability() == (9, 0), sparse_attention_config=sparse_config, - enable_ragged_verification=ragged, ) + dsa = metadata.ext(DeepseekV4Metadata) + # Ragged-verification buffers are sized when the extension allocates. + metadata.enable_ragged_verification = ragged + construct_extension(dsa) # Keep the persistent allocation ceiling independent of this # step's query lengths, as in update_spec_dec_param. metadata.max_draft_tokens = MAX_GENERATION_SEQ_LEN_Q - 1 @@ -1231,14 +1248,16 @@ def create_gen_metadata(ragged: bool) -> DeepseekV4TrtllmAttentionMetadata: device=device, ) torch.testing.assert_close( - gen_metadata.ragged_row_kv_lens(total_gen_tokens), expected_rows + gen_metadata.ragged_row_kv_lens(total_gen_tokens), + expected_rows, ) torch.testing.assert_close( gen_metadata.token_major_gen_view().sequence_length, expected_rows ) - gen_metadata.on_update_kv_lens() + refresh_from_lengths(gen_metadata) torch.testing.assert_close( - gen_metadata.ragged_row_kv_lens(total_gen_tokens), expected_rows + gen_metadata.ragged_row_kv_lens(total_gen_tokens), + expected_rows, ) torch.testing.assert_close( gen_metadata.token_major_gen_view().sequence_length, expected_rows @@ -1514,7 +1533,7 @@ def test_deepseek_v4_sparse_mla_mixed_batch(context_lengths: List[int]): ).uniform_(-1, 1) prefill_latent = torch.cat([prefill_compressed_kv, prefill_k_pe], dim=-1) - prefill_metadata = DeepseekV4TrtllmAttentionMetadata( + prefill_metadata = _create_metadata( seq_lens=torch.tensor(gen_ctx_lengths, dtype=torch.int), request_ids=gen_request_ids, max_num_requests=num_gen, @@ -1592,7 +1611,7 @@ def test_deepseek_v4_sparse_mla_mixed_batch(context_lengths: List[int]): # 4. Mixed metadata: request 0 = context, requests 1..N = generation. mixed_seq_lens = [context_lengths[0]] + [generation_seq_len_q] * num_gen mixed_cached_lens = [0, *gen_cached_lens] - mixed_metadata = DeepseekV4TrtllmAttentionMetadata( + mixed_metadata = _create_metadata( seq_lens=torch.tensor(mixed_seq_lens, dtype=torch.int), request_ids=request_ids, max_num_requests=batch_size, diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_indexer_slot_mapping.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_indexer_slot_mapping.py index c1886b7a8111..0e22b1785497 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_indexer_slot_mapping.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_indexer_slot_mapping.py @@ -7,17 +7,20 @@ import pytest import torch +from _torch.helpers import refresh_from_lengths from utils.util import skip_pre_blackwell from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import ( DeepseekV4CacheManager, - DeepseekV4TrtllmAttentionMetadata, + DeepseekV4TrtllmAttention, ) from tensorrt_llm._torch.attention.backends.sparse.dsa import ( DSACacheManagerV2, - DSAtrtllmAttentionMetadata, + DSAMetadata, + DSATrtllmAttention, ) from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import _fused_dsa_meta_enabled +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp @@ -37,7 +40,7 @@ def _prepared_metadata( *, context: bool, compressed: bool, -) -> Iterator[DSAtrtllmAttentionMetadata]: +) -> Iterator[TrtllmAttentionMetadata]: config_cls = DeepSeekV4SparseAttentionConfig if compressed else DeepSeekSparseAttentionConfig sparse_config = config_cls( index_n_heads=64, @@ -73,10 +76,8 @@ def _prepared_metadata( try: kv_lens = [cached + length for cached, length in zip(cached_tokens, seq_lens)] assert manager.add_dummy_requests([0, 1], kv_lens, is_gen=not context) is not None - metadata_cls = ( - DeepseekV4TrtllmAttentionMetadata if compressed else DSAtrtllmAttentionMetadata - ) - metadata = metadata_cls( + backend_cls = DeepseekV4TrtllmAttention if compressed else DSATrtllmAttention + metadata = backend_cls.Metadata( seq_lens=torch.tensor(seq_lens, dtype=torch.int32), request_ids=[0, 1], max_num_requests=2, @@ -147,7 +148,7 @@ def test_v4_runtime_maps_gather_physical_cache_bytes( metadata.slot_mapping_fp8.fill_(-1) metadata.slot_mapping_scale.fill_(-1) - metadata.on_update_kv_lens() + refresh_from_lengths(metadata) assert getattr(metadata, "_fused_dsa_meta_armed", False) is fused for offsets, width in ( @@ -192,6 +193,7 @@ def test_dsa_context_gather_alias_survives_runtime_refresh( context=True, compressed=False, ) as metadata: + dsa = metadata.ext(DSAMetadata) assert (metadata.slot_mapping_fp8_fullkv is metadata.slot_mapping_fp8) is (not cached) assert (metadata.slot_mapping_scale_fullkv is metadata.slot_mapping_scale) is (not cached) names = ( @@ -200,11 +202,11 @@ def test_dsa_context_gather_alias_survives_runtime_refresh( "slot_mapping_fp8_fullkv", "slot_mapping_scale_fullkv", ) - expected = [getattr(metadata, name).clone() for name in names] + expected = [getattr(dsa, name).clone() for name in names] - metadata.on_update_kv_lens() + refresh_from_lengths(metadata) assert (metadata.slot_mapping_fp8_fullkv is metadata.slot_mapping_fp8) is (not cached) assert (metadata.slot_mapping_scale_fullkv is metadata.slot_mapping_scale) is (not cached) for name, prepared in zip(names, expected): - torch.testing.assert_close(getattr(metadata, name), prepared, rtol=0, atol=0) + torch.testing.assert_close(getattr(dsa, name), prepared, rtol=0, atol=0) diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_ragged_fused_q_rope_specs.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_ragged_fused_q_rope_specs.py index d44330524748..0ef6e0e56655 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_ragged_fused_q_rope_specs.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_ragged_fused_q_rope_specs.py @@ -5,11 +5,12 @@ import pytest import torch +from _torch.helpers import bare_extension -from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( - DeepseekV4TrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import DeepseekV4Metadata from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.module import _fused_q_rope_specs +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata class _Mqa: @@ -20,6 +21,13 @@ def _ensure_rope_table_size(self, size): pass +def _serve_as_extension(metadata: SimpleNamespace) -> None: + """Let the stub act as both the core metadata and its DeepSeek-V4 extension.""" + metadata.metadata = metadata + metadata.find_ext = lambda cls: metadata + metadata.index_share = metadata + + def _ragged_metadata(verify_lens, *, context_lens=(), kv_lens=None): num_generations = len(verify_lens) if kv_lens is None: @@ -36,8 +44,9 @@ def _ragged_metadata(verify_lens, *, context_lens=(), kv_lens=None): fused_q_gen_cu_seqlens=torch.zeros(num_generations + 1, dtype=torch.int32), _fused_q_gen_cu_seqlens_valid=False, ) + _serve_as_extension(metadata) metadata.mla_prepare_fused_q_gen_cu_seqlens = MethodType( - DeepseekV4TrtllmAttentionMetadata.mla_prepare_fused_q_gen_cu_seqlens, metadata + DeepseekV4Metadata.mla_prepare_fused_q_gen_cu_seqlens, metadata ) return metadata @@ -68,6 +77,7 @@ def test_uniform_k5_keeps_scalar_generation_spec(): max_seq_len=1006, is_ragged_verify=False, ) + _serve_as_extension(metadata) _, specs = _fused_q_rope_specs(SimpleNamespace(mqa=_Mqa()), metadata, 0, 128) assert len(specs) == 1 rows, cache_lens, seq_len, cu_q_seqlens = specs[0] @@ -121,6 +131,7 @@ def test_ragged_generation_fails_closed_without_stable_prefix_hook(): max_seq_len=20, is_ragged_verify=True, ) + _serve_as_extension(metadata) mla = SimpleNamespace(mqa=_Mqa()) assert _fused_q_rope_specs(mla, metadata, 0, 2) == (None, []) metadata.mla_prepare_fused_q_gen_cu_seqlens = lambda: None @@ -150,7 +161,13 @@ def test_prefix_builder_fails_closed_without_graph_storage(capacity): def test_sequence_update_invalidates_cached_prefix(): - metadata = DeepseekV4TrtllmAttentionMetadata.__new__(DeepseekV4TrtllmAttentionMetadata) - metadata._fused_q_gen_cu_seqlens_valid = True - metadata._invalidate_mla_scheduler_buffers() - assert not metadata._fused_q_gen_cu_seqlens_valid + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) + dsv4 = DeepseekV4Metadata.__new__(DeepseekV4Metadata) + dsv4.metadata = metadata + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + metadata.replace_ext(dsv4) + dsv4._fused_q_gen_cu_seqlens_valid = True + dsv4._ragged_mla_buffers_valid = True + dsv4._invalidate_length_prefixes() + assert not dsv4._fused_q_gen_cu_seqlens_valid + assert not dsv4._ragged_mla_buffers_valid diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py index a80f5d7f5b21..c8951c7b9027 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py @@ -18,11 +18,13 @@ import pytest import torch +from _torch.helpers import bare_extension -from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( - DeepseekV4TrtllmAttentionMetadata, -) -from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.features import DraftMetadata +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import DeepseekV4Metadata +from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata class _StubCacheManager: @@ -34,6 +36,17 @@ def __init__(self, num_pools: int, max_blocks_per_seq: int) -> None: self.max_blocks_per_seq = max_blocks_per_seq +def _stub(extension_type: type = DSAMetadata) -> DSAMetadata: + """An extension built through ``__new__`` on a core metadata built the same way.""" + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) + extension = extension_type.__new__(extension_type) + extension.metadata = metadata + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + metadata.replace_ext(bare_extension(DraftMetadata, metadata)) + metadata.replace_ext(extension) + return extension + + def _metadata( *, num_pools: int, @@ -41,8 +54,8 @@ def _metadata( max_seqs: int, draft: int = 5, enable_ragged_verification: bool = True, -) -> DSAtrtllmAttentionMetadata: - """A metadata object with just enough state to build its own buffers. +) -> DSAMetadata: + """A DSA extension with just enough state to build its own buffers. Constructed through ``__new__`` rather than the real constructor: the real one wants a runtime, a resource manager and a model config, none of which @@ -50,17 +63,17 @@ def _metadata( new dependency shows up as a clear AttributeError here instead of being silently satisfied by a mock. """ - meta = DSAtrtllmAttentionMetadata.__new__(DSAtrtllmAttentionMetadata) - meta.kv_cache_manager = _StubCacheManager(num_pools, max_blocks) - meta.max_num_sequences = max_seqs - meta.max_num_requests = max_seqs + meta = _stub() + meta.metadata.kv_cache_manager = _StubCacheManager(num_pools, max_blocks) + meta.metadata.max_num_sequences = max_seqs + meta.metadata.max_num_requests = max_seqs meta.max_draft_tokens = draft meta.enable_ragged_verification = enable_ragged_verification - meta.cuda_graph_buffers = None + meta.metadata.cuda_graph_buffers = None meta.num_sms = torch.cuda.get_device_properties(0).multi_processor_count meta.runtime_tokens_per_gen_step = 0 meta.create_expanded_buffers(capture_graph=False) - meta.prompt_lens_cpu = torch.zeros(max_seqs, dtype=torch.int32) + meta.metadata.prompt_lens_cpu = torch.zeros(max_seqs, dtype=torch.int32) return meta @@ -81,12 +94,10 @@ def test_feature_off_skips_ragged_row_buffers() -> None: assert meta.attn_row_block_offsets is None -@pytest.mark.parametrize( - "metadata_type", [DSAtrtllmAttentionMetadata, DeepseekV4TrtllmAttentionMetadata] -) +@pytest.mark.parametrize("metadata_type", [DSAMetadata, DeepseekV4Metadata]) def test_feature_off_rejects_ragged_layout(metadata_type) -> None: """An unexpected ragged update fails instead of dereferencing absent buffers.""" - meta = metadata_type.__new__(metadata_type) + meta = _stub(metadata_type) meta.enable_ragged_verification = False meta.ragged_verify_lens = None assert not meta.is_ragged_verify @@ -96,37 +107,35 @@ def test_feature_off_rejects_ragged_layout(metadata_type) -> None: assert meta.is_ragged_verify with pytest.raises(RuntimeError, match="without enabling ragged-verification buffers"): - meta.prepare() + meta._validate_ragged_verification() with pytest.raises(RuntimeError, match="without enabling ragged-verification buffers"): meta.apply_device_ragged_layout( torch.tensor([2]), torch.tensor([0, 0]), torch.tensor([-1, 0]) ) -@pytest.mark.parametrize( - "metadata_type", [DSAtrtllmAttentionMetadata, DeepseekV4TrtllmAttentionMetadata] -) +@pytest.mark.parametrize("metadata_type", [DSAMetadata, DeepseekV4Metadata]) def test_device_window_kv_bound_preserves_context_and_input(metadata_type) -> None: - meta = metadata_type.__new__(metadata_type) + meta = _stub(metadata_type) meta.device_windows_mode = True meta.ragged_verify_lens = [2, 4] - meta._num_contexts = 1 - meta._seq_lens = torch.tensor([7, 2, 4], dtype=torch.int32) - meta._seq_lens_kv = None + meta.metadata._num_contexts = 1 + meta.metadata._seq_lens = torch.tensor([7, 2, 4], dtype=torch.int32) + meta.metadata._seq_lens_kv = None meta.max_draft_tokens = 5 - meta.kv_lens = torch.tensor([7, 102, 204], dtype=torch.int32) - original = meta.kv_lens.clone() + meta.metadata.kv_lens = torch.tensor([7, 102, 204], dtype=torch.int32) + original = meta.metadata.kv_lens.clone() bounded = meta._prepare_device_window_kv_lens(original) assert original.tolist() == [7, 102, 204] assert bounded.tolist() == [7, 106, 206] - assert meta.kv_lens.tolist() == bounded.tolist() + assert meta.metadata.kv_lens.tolist() == bounded.tolist() @pytest.mark.parametrize( "device_windows_mode,ragged_lens", [(False, [2]), (True, None), (False, None)] ) def test_native_kv_bound_reuses_storage(device_windows_mode, ragged_lens) -> None: - meta = DSAtrtllmAttentionMetadata.__new__(DSAtrtllmAttentionMetadata) + meta = _stub() meta.device_windows_mode = device_windows_mode meta.ragged_verify_lens = ragged_lens lengths = torch.tensor([102], dtype=torch.int32) @@ -134,7 +143,7 @@ def test_native_kv_bound_reuses_storage(device_windows_mode, ragged_lens) -> Non def _expand( - meta: DSAtrtllmAttentionMetadata, + meta: DSAMetadata, *, num_contexts: int, row_req_idx: list[int], @@ -143,14 +152,14 @@ def _expand( ) -> int: """Drive the private expansion with an explicit row->request map.""" ngt = len(row_req_idx) - meta.num_contexts = num_contexts + meta.metadata.num_contexts = num_contexts meta._ragged_num_rows = ngt meta.ragged_verify_lens = [1] * ngt - meta.kv_cache_block_offsets = block_offsets + meta.metadata.kv_cache_block_offsets = block_offsets meta.row_req_idx_host[:ngt].copy_(torch.tensor(row_req_idx, dtype=torch.long)) meta.row_kv_lens_host[:ngt].fill_(7) meta.row_kv_correction_host[:ngt].zero_() - meta.prompt_lens_cpu[:num_seqs].fill_(1) + meta.metadata.prompt_lens_cpu[:num_seqs].fill_(1) meta._prepare_token_major_gen_rows(torch.full((num_seqs,), 7, dtype=torch.int32)) return meta._attn_num_rows @@ -173,7 +182,7 @@ def test_row_block_table_matches_op_layout(num_pools: int) -> None: ) assert rows == len(row_req_idx) - got = meta.token_major_gen_view().kv_cache_block_offsets + got = meta.metadata.token_major_gen_view().kv_cache_block_offsets assert got.dim() == src.dim(), ( f"expanded table has rank {got.dim()} but the attention op reads rank " f"{src.dim()}; substituting it wholesale would misread every axis" @@ -240,13 +249,13 @@ def test_device_layout_preserves_mixed_context_row_and_token_prefixes() -> None: ) assert rows == 5 - meta.num_generations = 2 - meta._num_ctx_tokens = 5 - meta.seq_lens = torch.tensor([2, 3, 1, 2], dtype=torch.int32) - meta._seq_lens_cuda = meta.seq_lens.to(device="cuda") - meta.req_idx_per_token = torch.full((32,), -1, dtype=torch.int32, device="cuda") + meta.metadata.num_generations = 2 + meta.metadata._num_ctx_tokens = 5 + meta.metadata.seq_lens = torch.tensor([2, 3, 1, 2], dtype=torch.int32) + meta.metadata._seq_lens_cuda = meta.metadata.seq_lens.to(device="cuda") + meta.index_share.req_idx_per_token = torch.full((32,), -1, dtype=torch.int32, device="cuda") context_token_requests = torch.tensor([0, 0, 1, 1, 1], dtype=torch.int32, device="cuda") - meta.req_idx_per_token[:5].copy_(context_token_requests) + meta.index_share.req_idx_per_token[:5].copy_(context_token_requests) context_row_requests = torch.tensor([0, 1], dtype=torch.long, device="cuda") context_row_corrections = torch.tensor([10, 20], dtype=torch.int32, device="cuda") meta.attn_row_req_idx_cuda[:2].copy_(context_row_requests) @@ -266,11 +275,61 @@ def test_device_layout_preserves_mixed_context_row_and_token_prefixes() -> None: meta.attn_row_kv_correction_cuda[:5], torch.tensor([10, 20, -2, -1, 0], dtype=torch.int32, device="cuda"), ) - assert torch.equal(meta.req_idx_per_token[:5], context_token_requests) + assert torch.equal(meta.index_share.req_idx_per_token[:5], context_token_requests) assert torch.equal( - meta.req_idx_per_token[5:8], + meta.index_share.req_idx_per_token[5:8], torch.tensor([2, 3, 3], dtype=torch.int32, device="cuda"), ) + meta.metadata.check_lengths_current() + + +def test_device_layout_applies_kv_lens_delta_and_clears_lengths_changed(monkeypatch) -> None: + """Device-window reranking adjusts kv_lens_cuda, refreshes derived state, and leaves lengths clean.""" + torch.cuda.init() + meta = _metadata(num_pools=1, max_blocks=8, max_seqs=4) + block_offsets = torch.arange(1 * 4 * 2 * 8, dtype=torch.int32, device="cuda").reshape( + 1, 4, 2, 8 + ) + rows = _expand( + meta, + num_contexts=0, + row_req_idx=[0, 1, 1], + num_seqs=2, + block_offsets=block_offsets, + ) + assert rows == 3 + + md = meta.metadata + core = md.core() + core.num_generations = 2 + core._num_contexts = 0 + core._num_ctx_tokens = 0 + core._num_tokens = 3 + core.seq_lens = torch.tensor([2, 1], dtype=torch.int32) + core._seq_lens_cuda = torch.tensor([2, 1], dtype=torch.int32, device="cuda") + core.kv_lens = torch.tensor([104, 202], dtype=torch.int32) + core.kv_lens_cuda = torch.tensor([104, 202], dtype=torch.int32, device="cuda") + meta.index_share.req_idx_per_token = torch.full((32,), -1, dtype=torch.int32, device="cuda") + meta.indexer_k_cache_block_offsets = torch.arange( + 2 * 8, dtype=torch.int32, device="cuda" + ).reshape(2, 8) + refreshed = [] + monkeypatch.setattr(meta, "on_update_kv_lens", lambda: refreshed.append(True)) + + meta.apply_device_ragged_layout( + verify_lens=torch.tensor([1, 2], dtype=torch.int32, device="cuda"), + req_idx=torch.tensor([0, 1, 1], dtype=torch.long, device="cuda"), + kv_correction=torch.tensor([0, -1, 0], dtype=torch.int32, device="cuda"), + kv_lens_delta=torch.tensor([-2, 2], dtype=torch.int32, device="cuda"), + ) + + assert torch.equal(md.seq_lens_cuda[:2], torch.tensor([1, 2], dtype=torch.int32, device="cuda")) + assert torch.equal( + md.kv_lens_cuda[:2], torch.tensor([102, 204], dtype=torch.int32, device="cuda") + ) + assert refreshed == [True] + assert not md.lengths_changed + md.check_lengths_current() def test_every_runtime_view_shares_the_row_count() -> None: @@ -286,7 +345,7 @@ def test_every_runtime_view_shares_the_row_count() -> None: nc = 1 rows = _expand(meta, num_contexts=nc, row_req_idx=[0, 0, 1], num_seqs=6, block_offsets=src) - view = meta.token_major_gen_view() + view = meta.metadata.token_major_gen_view() for name in ( "sequence_length", "host_past_key_value_lengths", diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py index 06188216905e..0fa97a64e882 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py @@ -19,11 +19,8 @@ 1. fp8_mqa_logits operation from the DeepGEMM library 2. fp8_paged_mqa_logits operation with paged KV cache 3. compute_cu_seqlen_kv_bounds utility for batched causal attention -4. prepare/restore attention metadata for draft replay """ -import builtins -import functools import random from types import MethodType, SimpleNamespace from unittest.mock import Mock, patch @@ -32,6 +29,7 @@ import torch from tensorrt_llm import deep_gemm +from tensorrt_llm._torch.attention.backends.features import DraftMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionInputType, @@ -42,8 +40,8 @@ DSABackendForwardArgs, DSACacheManager, DSACacheManagerV2, + DSAMetadata, DSATrtllmAttention, - DSAtrtllmAttentionMetadata, Indexer, _effective_compress_ratio_divisor, _select_indexer_compress_ratio, @@ -69,16 +67,15 @@ DSAVanillaAttention, DSAVanillaIndexer, ) -from tensorrt_llm._torch.attention.backends.sparse.params import use_self_sampling_gvr +from tensorrt_llm._torch.attention.backends.sparse.params import ( + MTPIndexShareExtension, + use_self_sampling_gvr, +) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modules.multi_stream_utils import with_multi_stream from tensorrt_llm._torch.modules.top_k import TopK, TopKImplementation from tensorrt_llm._torch.pyexecutor._util import get_kv_cache_manager_cls from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import PageIndexMode, Role -from tensorrt_llm._torch.speculative.interface import ( - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, -) from tensorrt_llm._utils import get_sm_version from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.executor import KvCacheConfig as BindingKvCacheConfig @@ -95,8 +92,14 @@ __extra_import_path__ = ["..", "~/tests/unittest"] +from _torch.helpers import bare_extension, construct_extension, refresh_from_lengths from utils.util import check_accuracy, skip_pre_blackwell, skip_pre_hopper +from tensorrt_llm._torch.attention.backends.features.spec_decoding import ( + SpecDecodingConfig, + SpecDecodingMetadata, +) + from ._indexer_reference import mqa_logits, paged_mqa_logits @@ -132,6 +135,18 @@ def test_fp8_ds_mla_layout_treats_binding_config_as_auto(): assert storage_head_dim == 576 +def _dsa_stub() -> DSAMetadata: + """A DSA extension with its defaults, on a core metadata built without __init__.""" + md = object.__new__(TrtllmAttentionMetadata) + md.replace_ext(bare_extension(MTPIndexShareExtension, md)) + md.replace_ext(bare_extension(DraftMetadata, md)) + with patch.object(deep_gemm, "get_num_sms", return_value=1): + dsa = bare_extension(DSAMetadata, md) + md.replace_ext(dsa) + md.replace_ext(bare_extension(SpecDecodingMetadata, md)) + return dsa + + def _set_torch_top_k(indexer: Indexer) -> None: indexer.top_k = TopK( indexer.index_topk, @@ -532,6 +547,8 @@ def test_vanilla_local_topk_uses_trtllm_pool_coordinates(): _cached_req_idx_ctx=torch.tensor([0, 1], dtype=torch.int32), _cached_req_idx_gen=None, ) + # The stub serves as its own DSA extension. + metadata.ext = lambda cls: metadata local_topk = torch.tensor([[0, 65, -1], [63, 64, 130]], dtype=torch.int32) actual = DSAVanillaAttention._local_topk_to_global( @@ -635,21 +652,19 @@ def test_metadata_cache_geometry_comes_from_sparse_metadata_params(use_self_samp use_self_sampling_topk=use_self_sampling, ) sparse_metadata_params = sparse_config.to_sparse_metadata_params() - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.sparse_metadata_params = sparse_metadata_params - metadata.kv_cache_manager = SimpleNamespace( + metadata = _dsa_stub() + md = metadata.metadata + md.sparse_metadata_params = sparse_metadata_params + md.kv_cache_manager = SimpleNamespace( tokens_per_block=256, compressed_block_sizes={}, get_cache_indices=Mock(), ) - metadata.is_cuda_graph = False + md.is_cuda_graph = False metadata.create_buffers_for_mla_rope_append = Mock() metadata.create_buffers_for_indexer = Mock() with ( - patch( - "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata.TrtllmAttentionMetadata.__post_init__" - ), patch( "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata.IS_CUTLASS_DSL_AVAILABLE", True, @@ -659,7 +674,7 @@ def test_metadata_cache_geometry_comes_from_sparse_metadata_params(use_self_samp return_value=100, ), ): - DSAtrtllmAttentionMetadata.__post_init__(metadata) + construct_extension(metadata) assert metadata.indexer_head_dim == 96 assert metadata.compress_ratios == [1, 4, 128] @@ -689,26 +704,24 @@ def test_cute_dsl_paged_mqa_logits_gate(requested, cute_dsl_available, sm_100f, indexer_k_dtype="fp8", use_cute_dsl_paged_mqa_logits=requested, ) - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.sparse_metadata_params = sparse_config.to_sparse_metadata_params() - metadata.kv_cache_manager = SimpleNamespace( + metadata = _dsa_stub() + md = metadata.metadata + md.sparse_metadata_params = sparse_config.to_sparse_metadata_params() + md.kv_cache_manager = SimpleNamespace( tokens_per_block=256, compressed_block_sizes={}, get_cache_indices=Mock(), ) - metadata.is_cuda_graph = False + md.is_cuda_graph = False metadata.create_buffers_for_mla_rope_append = Mock() metadata.create_buffers_for_indexer = Mock() indexer_mod = "tensorrt_llm._torch.attention.backends.sparse.dsa.indexer" with ( - patch( - "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata.TrtllmAttentionMetadata.__post_init__" - ), patch(f"{indexer_mod}.IS_CUTLASS_DSL_AVAILABLE", cute_dsl_available), patch(f"{indexer_mod}.is_sm_100f", return_value=sm_100f), ): - DSAtrtllmAttentionMetadata.__post_init__(metadata) + construct_extension(metadata) indexer_decision = resolve_cute_dsl_paged_mqa_logits( sparse_config.to_sparse_params().use_cute_dsl_paged_mqa_logits ) @@ -845,6 +858,8 @@ def test_metadata_warmup_cute_dsl_radix_topk_dispatch( get_indexer_max_seq_len=Mock(return_value=32768), num_sms=148, ) + metadata.metadata = metadata + metadata.index_share = metadata with ( patch( @@ -855,7 +870,7 @@ def test_metadata_warmup_cute_dsl_radix_topk_dispatch( "tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops.warmup_cute_dsl_radix_topk_decode" ) as cute_dsl_radix, ): - DSAtrtllmAttentionMetadata.warmup_cute_dsl_radix_topk(metadata, next_n) + DSAMetadata.warmup_cute_dsl_radix_topk(metadata, next_n) if should_warmup: cute_dsl_radix.assert_called_once_with( @@ -896,6 +911,8 @@ def test_metadata_warmup_selfsampling_prefill_leg( use_cute_dsl_paged_mqa_logits=True, num_sms=148, ) + metadata.metadata = metadata + metadata.index_share = metadata ss_host = "tensorrt_llm._torch.kernels.blackwell.top_k.gvr_topk_decode_self_sampling_host" with ( patch( @@ -909,7 +926,7 @@ def test_metadata_warmup_selfsampling_prefill_leg( patch(f"{ss_host}.warmup_prefill") as warmup_prefill, patch(f"{ss_host}.warmup_varlen"), ): - DSAtrtllmAttentionMetadata.warmup_selfsampling_topk(metadata, next_n=1, batch_sizes=[8]) + DSAMetadata.warmup_selfsampling_topk(metadata, next_n=1, batch_sizes=[8]) if should_warmup: warmup_prefill.assert_called_once_with(512, max(msl_c, 32768)) @@ -925,7 +942,7 @@ def test_kv_lens_row_reorder_threshold(): def make_mock(num_generations, kv_lens_list): kv_lens_cuda = torch.tensor(kv_lens_list, dtype=torch.int32, device="cuda") row_order_buffer = torch.zeros(64, dtype=torch.int32, device="cuda") - return SimpleNamespace( + mock = SimpleNamespace( needs_gvr_prior=True, num_generations=num_generations, num_sms=num_sms, @@ -937,15 +954,17 @@ def make_mock(num_generations, kv_lens_list): kv_lens_row_reorder_buffer=row_order_buffer, kv_lens_row_reorder=None, ) + mock.metadata = mock + return mock kv_lens = [4, 1, 8, 2, 16, 3, 12, 6, 7, 9, 5, 11, 13, 10, 14, 15] metadata_below = make_mock(1, [1000]) - DSAtrtllmAttentionMetadata._compute_kv_lens_row_reorder(metadata_below) + DSAMetadata._compute_kv_lens_row_reorder(metadata_below) assert metadata_below.kv_lens_row_reorder is None metadata_at = make_mock(num_sms, kv_lens) - DSAtrtllmAttentionMetadata._compute_kv_lens_row_reorder(metadata_at) + DSAMetadata._compute_kv_lens_row_reorder(metadata_at) row_order = metadata_at.kv_lens_row_reorder.cpu().tolist() assert [kv_lens[i] for i in row_order] == sorted(kv_lens, reverse=True) @@ -1041,20 +1060,21 @@ def test_shared_topk_lifecycle(monkeypatch): ) assert sparse_metadata_params.has_shared_indexer_layers - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.sparse_metadata_params = sparse_metadata_params - metadata.max_num_sequences = 2 - metadata.max_num_tokens = 4 - metadata.num_sparse_topk = 3 + metadata = _dsa_stub() + md = metadata.metadata + md.sparse_metadata_params = sparse_metadata_params + md.max_num_sequences = 2 + md.max_num_tokens = 4 + md.num_sparse_topk = 3 metadata._radix_rows_per_sequence = 1 metadata.num_sms = 1 - metadata.cuda_graph_buffers = None - metadata.kv_cache_manager = SimpleNamespace(max_blocks_per_seq=2) - metadata.enable_context_mla_with_cached_kv = False + md.cuda_graph_buffers = None + md.kv_cache_manager = SimpleNamespace(max_blocks_per_seq=2) + md.enable_context_mla_with_cached_kv = False metadata.enable_indexer_skip = False metadata.enable_gvr_topk = False metadata.needs_gvr_prior = False - metadata.get_empty = Mock( + md.get_empty = Mock( side_effect=lambda _, shape, **kwargs: torch.empty(tuple(shape), dtype=kwargs["dtype"]) ) metadata._create_kv_lens_2d_buffer = Mock() @@ -1068,7 +1088,7 @@ def test_shared_topk_lifecycle(monkeypatch): ): metadata.create_buffers_for_indexer() - buffer = metadata.shared_topk_indices + buffer = metadata.index_share.shared_topk_indices assert buffer.shape == (4, 3) context_topk = torch.tensor([[0, 1, 2], [1, 2, 3]], dtype=torch.int32) @@ -1094,8 +1114,8 @@ def test_shared_topk_lifecycle(monkeypatch): backend._remap_topk_to_global = MethodType( DSATrtllmAttention._remap_topk_to_global, backend ) - metadata._num_ctx_tokens = context_topk.shape[0] - metadata._num_tokens = context_topk.shape[0] + generation_topk.shape[0] + md._num_ctx_tokens = context_topk.shape[0] + md._num_tokens = context_topk.shape[0] + generation_topk.shape[0] backend_args = DSABackendForwardArgs(indexer_intermediates=[]) def _predict(backend, input_type): @@ -1103,7 +1123,7 @@ def _predict(backend, input_type): backend, torch.empty((2, 1)), None, - metadata, + md, AttentionForwardArgs( attention_input_type=input_type, sparse_backend_args=backend_args, @@ -1118,7 +1138,7 @@ def _predict(backend, input_type): _predict(full_backend, AttentionInputType.context_only) _predict(full_backend, AttentionInputType.generation_only) torch.testing.assert_close( - metadata.shared_topk_indices, + metadata.index_share.shared_topk_indices, torch.cat([context_topk, generation_topk]), ) torch.testing.assert_close( @@ -1131,38 +1151,36 @@ def _predict(backend, input_type): ) metadata._invalidate_pool_view_cache = Mock() - metadata.kv_cache_manager = None - metadata._num_tokens = 0 - metadata._num_generations = 0 - metadata.kv_lens_cuda = torch.empty(0, dtype=torch.int32) + md.kv_cache_manager = None + md._num_tokens = 0 + md._num_generations = 0 + md.kv_lens_cuda = torch.empty(0, dtype=torch.int32) metadata.prepare_dense_topk_indices = Mock() - metadata.on_update_kv_lens() + refresh_from_lengths(md) - assert metadata.shared_topk_indices is buffer + assert metadata.index_share.shared_topk_indices is buffer @pytest.mark.parametrize("enabled", [False, True]) def test_kv_len_update_refreshes_ragged_rows_only_when_enabled(enabled): - metadata = DSAtrtllmAttentionMetadata.__new__(DSAtrtllmAttentionMetadata) - metadata.kv_cache_manager = None - metadata._num_tokens = 0 - metadata._num_generations = 0 + metadata = _dsa_stub() + md = metadata.metadata + md.kv_cache_manager = None + md._num_tokens = 0 + md._num_generations = 0 metadata.enable_ragged_verification = enabled metadata._invalidate_pool_view_cache = Mock() metadata._compute_kv_lens_row_reorder = Mock() metadata.prepare_dense_topk_indices = Mock() - metadata.kv_lens_cuda = None + md.kv_lens_cuda = None metadata.refresh_ragged_row_kv_lens = Mock() metadata.refresh_token_major_gen_rows = Mock() - with ( - patch.object(TrtllmAttentionMetadata, "on_update_kv_lens"), - patch( - "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata._fused_dsa_meta_enabled", - return_value=False, - ), + with patch( + "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata._fused_dsa_meta_enabled", + return_value=False, ): - metadata.on_update_kv_lens() + refresh_from_lengths(md) expected_calls = int(enabled) assert metadata.refresh_ragged_row_kv_lens.call_count == expected_calls @@ -1379,7 +1397,7 @@ def test_indexer_projection_dtype_follows_bf16_flag(monkeypatch, flag_value, exp ), ], ) -def test_metadata_spec_update_resizes_radix_workspace( +def test_metadata_sizes_radix_workspace_from_spec_decoding_config( is_tree, is_dynamic, max_draft_len, @@ -1387,35 +1405,24 @@ def test_metadata_spec_update_resizes_radix_workspace( spec_tree_manager, rows_per_sequence, ): - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.max_num_sequences = 2 + metadata = _dsa_stub() + md = metadata.metadata + md.max_num_sequences = 2 metadata.max_draft_tokens = 0 metadata._radix_rows_per_sequence = 1 - metadata.num_sparse_topk = 2 - metadata.is_cuda_graph = True - metadata.nvfp4_mla_fp8_scratch = None - metadata.kv_lens_cuda_2d = torch.empty(2, 1) - metadata.kv_lens_expanded_host = torch.empty(2) - metadata._create_kv_lens_2d_buffer = Mock() - metadata.create_expanded_buffers = Mock() - metadata._create_radix_aux_buffers = Mock() - - with patch( - "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata." - "TrtllmAttentionMetadata.update_spec_dec_param" - ): - metadata.update_spec_dec_param( - batch_size=2, - is_spec_decoding_enabled=True, - is_spec_dec_tree=is_tree, - is_spec_dec_dynamic_tree=is_dynamic, - max_draft_len=max_draft_len, - max_total_draft_tokens=max_total_draft_tokens, - spec_tree_manager=spec_tree_manager, - ) - + md.num_sparse_topk = 2 + md.is_cuda_graph = True + md.spec_decoding = SpecDecodingConfig( + is_spec_dec_tree=is_tree, + is_spec_dec_dynamic_tree=is_dynamic, + max_draft_len=max_draft_len, + max_total_draft_tokens=max_total_draft_tokens, + tree_buffer_dim=getattr(spec_tree_manager, "_internal_buf_dim", 0), + ) + metadata._size_for_spec_decoding() + + assert metadata.max_draft_tokens == max_draft_len assert metadata._radix_rows_per_sequence == rows_per_sequence - metadata._create_radix_aux_buffers.assert_called_once_with(True) def create_dsa_cache_manager( @@ -1510,6 +1517,9 @@ def test_transform_local_topk_uses_v2_page_mapping(): _cached_tokens_per_block=4, _cached_pool_view=torch.empty(0), ) + # The stub serves as its own DSA extension. + metadata.ext = lambda cls: metadata + metadata.index_share = metadata expected_indices = torch.tensor([[90]], dtype=torch.int32) with patch( @@ -1576,13 +1586,17 @@ def _make_group_remap_metadata( _cached_tokens_per_block=block_size, _cached_pool_view=torch.empty(0, device=device), ) + # The stub serves as both the core metadata and its DSA extension. + metadata.metadata = metadata + metadata.ext = lambda cls: metadata + metadata.index_share = metadata metadata._ensure_group_remap_struct = MethodType( - DSAtrtllmAttentionMetadata._ensure_group_remap_struct, metadata + DSAMetadata._ensure_group_remap_struct, metadata ) # Bind the real per-step cache invalidation so tests exercise the actual # clear of _group_remap_batched at forward boundaries. metadata._invalidate_pool_view_cache = MethodType( - DSAtrtllmAttentionMetadata._invalidate_pool_view_cache, metadata + DSAMetadata._invalidate_pool_view_cache, metadata ) return metadata, block_table, req_idx @@ -2274,8 +2288,18 @@ def __init__(self): """Initialize with the provided num_cached_tokens.""" self.num_cached_tokens_per_seq = num_cached_tokens - class MockMetadata(DSAtrtllmAttentionMetadata): - """Mock DSA attention metadata for unit testing indexer operations.""" + class MockMetadata(TrtllmAttentionMetadata): + """Mock attention metadata with a DSA extension for unit testing indexer operations.""" + + _instance_attrs = frozenset( + { + "_num_seqs", + "ctx_kv_offsets", + "gen_kv_offsets", + "num_gen_tokens", + "scheduler_metadata_buffer_mtp3", + } + ) def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: """Initialize mock metadata with buffers, block tables, and indexer state. @@ -2286,21 +2310,23 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: GC would free the manager -- on whatever thread runs the collection, which need not have a CUDA context. """ - self.num_sms = deep_gemm.get_num_sms() + self.replace_ext(bare_extension(MTPIndexShareExtension, self)) + dsa = bare_extension(DSAMetadata, self) + dsa.num_sms = deep_gemm.get_num_sms() self.max_num_tokens = max_num_tokens self.request_ids = request_ids self.num_contexts = num_contexts self.num_generations = num_generations self._num_seqs = num_contexts + num_generations self.max_num_sequences = batch_size - self.max_draft_tokens = max_draft_tokens + dsa.max_draft_tokens = max_draft_tokens self.num_sparse_topk = index_topk - self._radix_rows_per_sequence = 1 + self.max_draft_tokens - self._create_radix_aux_buffers(capture_graph=False) - self.enable_indexer_skip = enable_indexer_skip - self.indexer_head_dim = indexer_head_dim - self.indexer_quant_block_size = 128 - self.compress_ratios = [compress_ratio] + dsa._radix_rows_per_sequence = 1 + dsa.max_draft_tokens + dsa._create_radix_aux_buffers(capture_graph=False) + dsa.enable_indexer_skip = enable_indexer_skip + dsa.indexer_head_dim = indexer_head_dim + dsa.indexer_quant_block_size = 128 + dsa.compress_ratios = [compress_ratio] # Keep seq_lens on CPU for split_prefill_chunks and other CPU operations # CUDA kernels will convert to CUDA as needed self.seq_lens = seq_lens.cpu() if seq_lens.is_cuda else seq_lens @@ -2308,36 +2334,36 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: self.kv_cache_params = MockKVCacheParams() self.kv_cache_manager = cache_manager # Effective tokens-per-block for the indexer k-cache slot mapping, - # mirroring DSAtrtllmAttentionMetadata.__post_init__ (dsa.py). + # mirroring DSAMetadata.allocate. tpb = self.kv_cache_manager.tokens_per_block - self._indexer_compress_ratio = _select_indexer_compress_ratio(self.compress_ratios) + dsa._indexer_compress_ratio = _select_indexer_compress_ratio(dsa.compress_ratios) if hasattr(self.kv_cache_manager, "compressed_block_sizes"): - tpb = tpb // _effective_compress_ratio_divisor(self._indexer_compress_ratio) - self._tokens_per_block = tpb + tpb = tpb // _effective_compress_ratio_divisor(dsa._indexer_compress_ratio) + dsa._tokens_per_block = tpb self.kv_lens_cuda_runtime = kv_lens.cuda() - self.indexer_k_cache_block_offsets = torch.zeros( + dsa.indexer_k_cache_block_offsets = torch.zeros( [batch_size, self.kv_cache_manager.max_blocks_per_seq], device="cuda", dtype=torch.int32, ) - self.host_indexer_k_cache_block_offsets = torch.zeros_like( - self.indexer_k_cache_block_offsets, + dsa.host_indexer_k_cache_block_offsets = torch.zeros_like( + dsa.indexer_k_cache_block_offsets, device="cpu", pin_memory=True, ) block_ids = self.kv_cache_manager.get_batch_cache_indices(request_ids) for i in range(len(block_ids)): - self.host_indexer_k_cache_block_offsets[i, : len(block_ids[i])] = torch.tensor( + dsa.host_indexer_k_cache_block_offsets[i, : len(block_ids[i])] = torch.tensor( block_ids[i], dtype=torch.int32 ) - self.indexer_k_cache_block_offsets[:batch_size].copy_( - self.host_indexer_k_cache_block_offsets[:batch_size], non_blocking=True + dsa.indexer_k_cache_block_offsets[:batch_size].copy_( + dsa.host_indexer_k_cache_block_offsets[:batch_size], non_blocking=True ) - self.slot_mapping_fp8 = torch.zeros((num_tokens,), device="cuda", dtype=torch.int64) - self.slot_mapping_scale = torch.zeros((num_tokens,), device="cuda", dtype=torch.int64) - self.scheduler_metadata_buffer = torch.zeros( - (self.num_sms + 1, 2), device="cuda", dtype=torch.int32 + dsa.slot_mapping_fp8 = torch.zeros((num_tokens,), device="cuda", dtype=torch.int64) + dsa.slot_mapping_scale = torch.zeros((num_tokens,), device="cuda", dtype=torch.int64) + dsa.scheduler_metadata_buffer = torch.zeros( + (dsa.num_sms + 1, 2), device="cuda", dtype=torch.int32 ) # DSL needs the next_n=1 schedule preserved (kNumNextNAtoms=1), # so allocate a separate buffer for the full-next_n schedule. @@ -2345,37 +2371,36 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: # `scheduler_metadata_buffer` itself (the alias makes # `Indexer.prepare()`'s second populate overwrite the first). if use_cute_dsl_paged_mqa_logits: - self.scheduler_metadata_buffer_full_next_n = torch.zeros( - (self.num_sms + 1, 2), device="cuda", dtype=torch.int32 + dsa.scheduler_metadata_buffer_full_next_n = torch.zeros( + (dsa.num_sms + 1, 2), device="cuda", dtype=torch.int32 ) else: - self.scheduler_metadata_buffer_full_next_n = self.scheduler_metadata_buffer - # Pre-allocated 2D kv_lens buffer for the DeepGEMM 2D context_lens API. - self.kv_lens_cuda_2d = torch.zeros( - (self.num_seqs, 1 + self.max_draft_tokens), device="cuda", dtype=torch.int32 - ) + dsa.scheduler_metadata_buffer_full_next_n = dsa.scheduler_metadata_buffer + # The 2D kv_lens buffer of the DeepGEMM 2D context_lens API and the + # compressed generation indexer KV lengths, as DSAMetadata allocates them. + dsa._create_kv_lens_2d_buffer(capture_graph=False) if num_generations > 0: gen_kv_lens = ( kv_lens[num_contexts : num_contexts + num_generations].cuda().to(torch.int32) ) - next_n_cap = 1 + self.max_draft_tokens - self.kv_lens_cuda_2d[:num_generations, :next_n_cap].copy_( + next_n_cap = 1 + dsa.max_draft_tokens + dsa.kv_lens_cuda_2d[:num_generations, :next_n_cap].copy_( gen_kv_lens.unsqueeze(-1).expand(-1, next_n_cap) ) - self.cu_seqlen_ks = torch.zeros((num_tokens,), device="cuda", dtype=torch.int32) - self.cu_seqlen_ke = torch.zeros((num_tokens,), device="cuda", dtype=torch.int32) + dsa.cu_seqlen_ks = torch.zeros((num_tokens,), device="cuda", dtype=torch.int32) + dsa.cu_seqlen_ke = torch.zeros((num_tokens,), device="cuda", dtype=torch.int32) self.ctx_kv_offsets = torch.zeros((num_tokens, 1), device="cuda", dtype=torch.int32) self.gen_kv_offsets = torch.zeros((num_tokens, 1), device="cuda", dtype=torch.int32) - self.host_slot_mapping_fp8 = torch.zeros_like( - self.slot_mapping_fp8, device="cpu", pin_memory=True + dsa.host_slot_mapping_fp8 = torch.zeros_like( + dsa.slot_mapping_fp8, device="cpu", pin_memory=True ) - self.host_slot_mapping_scale = torch.zeros_like( - self.slot_mapping_scale, device="cpu", pin_memory=True + dsa.host_slot_mapping_scale = torch.zeros_like( + dsa.slot_mapping_scale, device="cpu", pin_memory=True ) self.host_ctx_kv_indptr = torch.zeros( (num_contexts + 1,), device="cpu", pin_memory=True, dtype=torch.int64 ) - self.host_gen_kv_indptr = torch.zeros( + dsa.host_gen_kv_indptr = torch.zeros( (num_generations + 1,), device="cpu", pin_memory=True, dtype=torch.int64 ) # Add host_ctx_cached_token_indptr for prepare_one_prefill_chunk @@ -2385,7 +2410,7 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: self._num_ctx_tokens = num_ctx_tokens self._num_tokens = num_tokens self.num_gen_tokens = num_tokens - num_ctx_tokens - # Also set private attributes used by DSAtrtllmAttentionMetadata + # Also set private attributes used by TrtllmAttentionMetadata self._num_contexts = num_contexts self._num_generations = num_generations self._num_ctx_tokens = num_ctx_tokens @@ -2401,7 +2426,7 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: kv_lens[num_contexts : num_contexts + num_generations], dim=0, dtype=torch.int64, - out=self.host_gen_kv_indptr[1 : num_generations + 1], + out=dsa.host_gen_kv_indptr[1 : num_generations + 1], ) # Compute host_ctx_cached_token_indptr from num_cached_tokens if num_contexts > 0: @@ -2414,8 +2439,8 @@ def __init__(self, cache_manager: DSACacheManager | DSACacheManagerV2) -> None: ) # Add indexer-specific attributes - self.indexer_max_chunk_size = indexer_max_chunk_size - self.indexer_prefill_chunks = None + dsa.indexer_max_chunk_size = indexer_max_chunk_size + dsa.indexer_prefill_chunks = None self.enable_context_mla_with_cached_kv = enable_context_mla_with_cached_kv @@ -2434,47 +2459,47 @@ def __init__(self): self.runtime_features = RuntimeFeatures() # Add expanded buffers for MTP support. Mirrors the production gate in - # DSAtrtllmAttentionMetadata.prepare_for_spec_decode: sm100+ DeepGEMM + # DSAMetadata.prepare_for_spec_decode: sm100+ DeepGEMM # runs a native next_n for any MTP depth, so only sm90 expands. The DSL # kernel supports arbitrary next_n natively, so it never needs expansion. - self.use_expanded_buffers_for_mtp = ( + dsa.use_expanded_buffers_for_mtp = ( not use_cute_dsl_paged_mqa_logits - and self.max_draft_tokens > 1 + and dsa.max_draft_tokens > 1 and get_sm_version() == 90 ) - self.kv_lens_expanded_cuda = torch.zeros( - (self.num_seqs * (1 + self.max_draft_tokens),), device="cuda", dtype=torch.int32 + dsa.kv_lens_expanded_cuda = torch.zeros( + (self.num_seqs * (1 + dsa.max_draft_tokens),), device="cuda", dtype=torch.int32 ) - self.kv_lens_expanded_host = torch.zeros_like( - self.kv_lens_expanded_cuda, device="cpu", pin_memory=True + dsa.kv_lens_expanded_host = torch.zeros_like( + dsa.kv_lens_expanded_cuda, device="cpu", pin_memory=True ) - self.block_table_expanded = torch.zeros( + dsa.block_table_expanded = torch.zeros( ( - self.num_seqs * (1 + self.max_draft_tokens), + self.num_seqs * (1 + dsa.max_draft_tokens), self.kv_cache_manager.max_blocks_per_seq, ), device="cuda", dtype=torch.int32, ) - self.host_block_table_expanded = torch.zeros_like( - self.block_table_expanded, device="cpu", pin_memory=True + dsa.host_block_table_expanded = torch.zeros_like( + dsa.block_table_expanded, device="cpu", pin_memory=True ) - self.scheduler_metadata_buffer_expanded = torch.zeros( - (self.num_sms + 1, 2), device="cuda", dtype=torch.int32 + dsa.scheduler_metadata_buffer_expanded = torch.zeros( + (dsa.num_sms + 1, 2), device="cuda", dtype=torch.int32 ) - if self.max_draft_tokens == 3: + if dsa.max_draft_tokens == 3: self.scheduler_metadata_buffer_mtp3 = torch.zeros( - (self.num_sms // 2 + 1, 2), device="cuda", dtype=torch.int32 + (dsa.num_sms // 2 + 1, 2), device="cuda", dtype=torch.int32 ) - if self.use_expanded_buffers_for_mtp: + if dsa.use_expanded_buffers_for_mtp: gen_kv_lens = kv_lens[num_contexts : self.num_seqs] gen_kv_lens_expanded = torch.stack( - [gen_kv_lens] * (1 + self.max_draft_tokens), dim=0 + [gen_kv_lens] * (1 + dsa.max_draft_tokens), dim=0 ) gen_kv_lens_expanded = gen_kv_lens_expanded.transpose(0, 1).contiguous().flatten() - self.kv_lens_expanded_host[: self.num_gen_tokens].copy_(gen_kv_lens_expanded) - self.kv_lens_expanded_cuda[: self.num_gen_tokens].copy_( - self.kv_lens_expanded_host[: self.num_gen_tokens], non_blocking=True + dsa.kv_lens_expanded_host[: self.num_gen_tokens].copy_(gen_kv_lens_expanded) + dsa.kv_lens_expanded_cuda[: self.num_gen_tokens].copy_( + dsa.kv_lens_expanded_host[: self.num_gen_tokens], non_blocking=True ) if self.kv_cache_manager is not None: @@ -2483,39 +2508,40 @@ def __init__(self): if len(gen_block_ids) > 0: # Find max length and create padded tensor max_len = max(len(bid) for bid in gen_block_ids) - gen_block_tensor = self.host_indexer_k_cache_block_offsets[ + gen_block_tensor = dsa.host_indexer_k_cache_block_offsets[ self.num_contexts : self.num_seqs, :max_len ] expanded_blocks = gen_block_tensor.repeat_interleave( - 1 + self.max_draft_tokens, dim=0 + 1 + dsa.max_draft_tokens, dim=0 ) - self.host_block_table_expanded[: self.num_gen_tokens, :max_len].copy_( + dsa.host_block_table_expanded[: self.num_gen_tokens, :max_len].copy_( expanded_blocks, non_blocking=True ) - self.block_table_expanded[: self.num_gen_tokens].copy_( - self.host_block_table_expanded[: self.num_gen_tokens], non_blocking=True + dsa.block_table_expanded[: self.num_gen_tokens].copy_( + dsa.host_block_table_expanded[: self.num_gen_tokens], non_blocking=True ) # Add skip indexer attributes - self.topk_indices_buffer = torch.zeros( + dsa.topk_indices_buffer = torch.zeros( (num_tokens, self.num_sparse_topk), device="cuda", dtype=torch.int32 ) - if self.num_contexts > 0 and self.enable_indexer_skip: - self.skip_indexer_for_ctx_reqs = ( + if self.num_contexts > 0 and dsa.enable_indexer_skip: + dsa.skip_indexer_for_ctx_reqs = ( kv_lens[: self.num_contexts].max().item() <= self.num_sparse_topk ) else: - self.skip_indexer_for_ctx_reqs = False + dsa.skip_indexer_for_ctx_reqs = False - if self.num_generations > 0 and self.enable_indexer_skip: - self.max_draft_tokens + 1 - self.skip_indexer_for_gen_reqs = ( + if self.num_generations > 0 and dsa.enable_indexer_skip: + dsa.max_draft_tokens + 1 + dsa.skip_indexer_for_gen_reqs = ( kv_lens[self.num_contexts : self.num_seqs].max().item() <= self.num_sparse_topk ) else: - self.skip_indexer_for_gen_reqs = False - self.prepare_dense_topk_indices(self.kv_lens_cuda_runtime, device=True) + dsa.skip_indexer_for_gen_reqs = False + dsa.prepare_dense_topk_indices(self.kv_lens_cuda_runtime, device=True) + self.replace_ext(dsa) @property def num_seqs(self) -> int: @@ -2529,9 +2555,10 @@ def num_seqs(self) -> int: @pytest.mark.parametrize("disabled_ratio", [0, 1]) def test_indexer_compress_ratio_zero_or_one_means_uncompressed(disabled_ratio): - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.kv_cache_manager = Mock() - metadata.kv_cache_manager.max_seq_len = 4096 + metadata = _dsa_stub() + md = metadata.metadata + md.core().kv_cache_manager = Mock() + md.kv_cache_manager.max_seq_len = 4096 kv_lens = torch.tensor([0, 1, 128, 4096], dtype=torch.int32) metadata._indexer_compress_ratio = disabled_ratio @@ -2898,7 +2925,7 @@ def test_fp8_k_cache_roundtrip(dsa_cache_manager): @skip_pre_hopper # next_n=5 (MTP-4) exercises the native paged-MQA path on sm100+ (no buffer # expansion); on sm90 it falls back to the expanded buffers. See -# DSAtrtllmAttentionMetadata.prepare_for_spec_decode. +# DSAMetadata.prepare_for_spec_decode. @pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (4, 3), (4, 4), (2, 5)]) @pytest.mark.parametrize( "backend", @@ -3309,7 +3336,7 @@ def _force_direct_path(meta): ) def _force_dsl_expand_setup(meta): - """Mirror DSAtrtllmAttentionMetadata.prepare's `expand_for_dsl` + """Mirror DSAMetadata.prepare's `expand_for_dsl` populate so Indexer.prepare can build the schedule from kv_lens_expanded_cuda. Calls the wave-aware picker; expand only happens when factor > 1. @@ -5434,129 +5461,6 @@ def fake_allgather(tensor, mapping, dim=-1, sizes=None): assert min_similarity == 1.0, f"TP split vs unsplit select different indices: {min_similarity}" -@pytest.mark.cpu_only -class TestPrepareRestoreAttnMetadataForDraftReplay: - """Tests for prepare_attn_metadata_for_draft_replay and - restore_attn_metadata_after_draft_replay.""" - - @staticmethod - def _make_mock_metadata(): - """Create a mock attention metadata object with KV cache block offsets.""" - meta = Mock() - meta.enable_flash_mla = False - meta.kv_cache_manager = Mock(name="target_kv_cache_manager") - meta.kv_cache_block_offsets = torch.tensor([10, 20, 30]) - meta.host_kv_cache_block_offsets = torch.tensor([10, 20, 30]) - meta.draft_kv_cache_block_offsets = torch.tensor([100, 200, 300]) - meta.prepare_for_draft_forward.return_value = None - meta.draft_replay_swapped_attrs = {} - for helper in ("record_draft_swap", "swap_for_draft", "restore_draft_swaps"): - setattr(meta, helper, functools.partial(getattr(TrtllmAttentionMetadata, helper), meta)) - return meta - - @staticmethod - def _make_mock_draft_manager(): - """Create a mock draft KV cache manager with host block offsets.""" - return SimpleNamespace(host_kv_cache_block_offsets=torch.tensor([100, 200, 300])) - - def test_prepare_swaps_and_restore_recovers(self): - """Test that prepare swaps KV manager and restore recovers original state.""" - meta = self._make_mock_metadata() - mgr = self._make_mock_draft_manager() - # DeepSeek-V4 metadata inherits DSA metadata, while its manager exposes - # index_head_dim without implementing DSA's full page-mapping contract. - mgr.index_head_dim = 128 - mgr.get_pool_block_indices = Mock() - mgr.indexer_k_cache_page_scale = 1 - original_kv_mgr = meta.kv_cache_manager - original_offsets = meta.kv_cache_block_offsets.clone() - original_host_offsets = meta.host_kv_cache_block_offsets.clone() - - def is_attn_metadata(obj, cls): - if cls in (TrtllmAttentionMetadata, DSAtrtllmAttentionMetadata): - return obj is meta - return builtins.isinstance(obj, cls) - - with patch( - "tensorrt_llm._torch.speculative.interface.isinstance", - side_effect=is_attn_metadata, - ): - saved = prepare_attn_metadata_for_draft_replay(meta, mgr) - - assert saved is not None - assert meta.draft_replay_swapped_attrs["kv_cache_manager"] is original_kv_mgr - assert meta.kv_cache_manager is mgr - assert "saved_backend_state" not in saved - meta.prepare_for_draft_forward.assert_called_once_with() - - restore_attn_metadata_after_draft_replay(meta, saved) - - assert meta.kv_cache_manager is original_kv_mgr - torch.testing.assert_close(meta.kv_cache_block_offsets, original_offsets) - torch.testing.assert_close(meta.host_kv_cache_block_offsets, original_host_offsets) - meta.restore_after_draft_forward.assert_called_once_with(None) - - def test_native_dsa_replay_swaps_and_restores_buffers(self): - """Switch native DSA metadata to draft buffers and restore it.""" - meta = self._make_mock_metadata() - mgr = object.__new__(DSACacheManagerV2) - mgr.host_kv_cache_block_offsets = torch.tensor([100, 200, 300]) - meta.enable_context_mla_with_cached_kv = False - meta._invalidate_pool_view_cache = Mock() - buffer_pairs = { - "host_indexer_k_cache_block_offsets": "host_draft_indexer_k_cache_block_offsets", - "indexer_k_cache_block_offsets": "draft_indexer_k_cache_block_offsets", - "host_slot_mapping_fp8": "host_draft_slot_mapping_fp8", - "host_slot_mapping_scale": "host_draft_slot_mapping_scale", - "slot_mapping_fp8": "draft_slot_mapping_fp8", - "slot_mapping_scale": "draft_slot_mapping_scale", - "block_table": "draft_block_table", - "block_table_expanded": "draft_block_table_expanded", - "host_block_table_expanded": "host_draft_block_table_expanded", - } - target_buffers = {} - draft_buffers = {} - for target_name, draft_name in buffer_pairs.items(): - target_buffers[target_name] = object() - draft_buffers[target_name] = object() - setattr(meta, target_name, target_buffers[target_name]) - setattr(meta, draft_name, draft_buffers[target_name]) - del meta.slot_mapping_fp8_fullkv - del meta.slot_mapping_scale_fullkv - - meta.prepare_for_draft_forward.side_effect = ( - lambda: DSAtrtllmAttentionMetadata.prepare_for_draft_forward(meta) - ) - meta.restore_after_draft_forward.side_effect = ( - lambda saved_state: DSAtrtllmAttentionMetadata.restore_after_draft_forward( - meta, saved_state - ) - ) - - def is_attn_metadata(obj, cls): - if cls in (TrtllmAttentionMetadata, DSAtrtllmAttentionMetadata): - return obj is meta - return builtins.isinstance(obj, cls) - - with ( - patch( - "tensorrt_llm._torch.speculative.interface.isinstance", - side_effect=is_attn_metadata, - ), - patch( - "tensorrt_llm._torch.attention.backends.sparse.dsa.metadata.torch.cuda.is_current_stream_capturing", - return_value=True, - ), - ): - saved = prepare_attn_metadata_for_draft_replay(meta, mgr) - - assert "saved_backend_state" in saved - assert meta.slot_mapping_fp8 is draft_buffers["slot_mapping_fp8"] - restore_attn_metadata_after_draft_replay(meta, saved) - assert meta.slot_mapping_fp8 is target_buffers["slot_mapping_fp8"] - assert meta.slot_mapping_fp8_fullkv is target_buffers["slot_mapping_fp8"] - - @pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available") @skip_pre_blackwell def test_cutedsl_mqa_logits_output_buffer_persistent(dsa_cache_manager): @@ -5731,6 +5635,15 @@ def _seam_indexer(): return indexer +def _seam_metadata(**fields): + """A metadata stub that is also its own DSA extension and index-share state.""" + metadata = SimpleNamespace(**fields) + metadata.ext = lambda extension_type: metadata + metadata.index_share = metadata + metadata.bind_buffers = lambda **buffers: vars(metadata).update(buffers) + return metadata + + def test_reused_topk_copy_is_the_row_copy(): gen = torch.Generator().manual_seed(0) shared = torch.randint(0, 1000, (4, 16), generator=gen, dtype=torch.int32) @@ -5739,7 +5652,7 @@ def test_reused_topk_copy_is_the_row_copy(): expected[2:5, :] = shared[:3, :] _seam_indexer()._copy_reused_topk( - SimpleNamespace(shared_topk_indices=shared), buffer, 2, 3, 3, False, None + _seam_metadata(shared_topk_indices=shared), buffer, 2, 3, 3, False, None ) assert torch.equal(buffer, expected) @@ -5748,7 +5661,7 @@ def test_reused_topk_copy_is_the_row_copy(): def test_fill_decode_topk_leaves_the_rows_to_the_decode_branch(): buffer = torch.full((6, 8), -1, dtype=torch.int32) - filled = _seam_indexer()._fill_decode_topk(SimpleNamespace(), None, None, buffer, 0, 6) + filled = _seam_indexer()._fill_decode_topk(_seam_metadata(), None, None, buffer, 0, 6) assert filled is False assert bool((buffer == -1).all()) @@ -5804,7 +5717,7 @@ def test_mtp_shared_rows_are_the_inline_statements( ) def metadata(): - return SimpleNamespace( + return _seam_metadata( seq_lens_cuda=seq_lens.clone(), mtp_num_accepted=None if accepted is None else torch.tensor(accepted), shared_topk_indices=(torch.full((8, 8), -7, dtype=torch.int32) if preset else None), diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_sparse_mla.py b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_sparse_mla.py index d2e457aba4ff..7fe5ad5db8d5 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_sparse_mla.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_sparse_mla.py @@ -26,6 +26,8 @@ from utils.util import skip_pre_blackwell import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.attention.backends.interface import ( AttentionInputType, MLAParams, @@ -703,7 +705,9 @@ def create_layer(layer_idx: int, num_kv_heads: int): kv_cache_manager.add_dummy_requests(request_ids, context_sequence_lengths) # Both backends use the DSA metadata: the vanilla backend runs the same # indexer plumbing and only swaps the selection kernels for torch. - metadata_sparse_kwargs = {"sparse_attention_config": sparse_config} + metadata_sparse_kwargs = { + "sparse_metadata_params": sparse_config.to_sparse_metadata_params() + } ctx_seq_lens = torch.tensor(context_sequence_lengths, dtype=torch.int) total_ctx_tokens = sum(context_sequence_lengths) @@ -750,22 +754,16 @@ def create_layer(layer_idx: int, num_kv_heads: int): ], ), mapping=mapping, - enable_flash_mla=torch.cuda.get_device_capability() == (9, 0), + extension_types=(FlashMlaMetadata,) + if torch.cuda.get_device_capability() == (9, 0) + else (), + spec_decoding=SpecDecodingConfig( + max_draft_len=generation_seq_len_q - 1, + max_total_draft_tokens=generation_seq_len_q - 1, + ), **metadata_sparse_kwargs, ) - attn_metadata.update_spec_dec_param( - batch_size=max_num_contexts, - is_spec_decoding_enabled=False, - is_spec_dec_tree=False, - is_spec_dec_dynamic_tree=False, - max_draft_len=generation_seq_len_q - 1, - max_total_draft_tokens=generation_seq_len_q - 1, - ) attn_metadata.prepare() - # The executor refreshes device-side metadata after prepare() on - # every step; the CuTe DSL paged-MQA atom split reads its - # expanded schedule from this refresh. - attn_metadata.on_update_kv_lens() for layer_idx in range(num_layers): print(f"---- step {step} layer {layer_idx} start ----") if step == 0: diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_metadata_topk_init.py b/tests/unittest/_torch/attention/sparse/dsa/test_metadata_topk_init.py index df1200650c45..c179c05d1463 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_metadata_topk_init.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_metadata_topk_init.py @@ -7,9 +7,27 @@ import pytest import torch +from _torch.helpers import bare_extension, construct_extension +from tensorrt_llm._torch.attention.backends.features import DraftMetadata from tensorrt_llm._torch.attention.backends.sparse.dsa import metadata as dsa_metadata from tensorrt_llm._torch.attention.backends.sparse.dsa.params import DSAMetadataParams +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + + +def _dsa_without_init( + md: TrtllmAttentionMetadata, params: DSAMetadataParams +) -> dsa_metadata.DSAMetadata: + """A DSA extension of ``md`` with only the defaults allocate() reads.""" + md.replace_ext(bare_extension(MTPIndexShareExtension, md)) + dsa = object.__new__(dsa_metadata.DSAMetadata) + dsa.metadata = md + dsa.sparse_metadata_params = params + dsa.enable_ragged_verification = False + dsa.ragged_verify_lens = None + md.replace_ext(dsa) + return dsa @pytest.mark.parametrize("enable_heuristic", [False, True]) @@ -22,8 +40,8 @@ def test_topk_flags_initialized_before_buffer_allocation( dsl_available: bool, sm_version: int, ) -> None: - metadata = object.__new__(dsa_metadata.DSAtrtllmAttentionMetadata) - metadata.sparse_metadata_params = DSAMetadataParams( + md = object.__new__(TrtllmAttentionMetadata) + params = DSAMetadataParams( indexer_max_chunk_size=8192, max_sparse_topk=512, index_head_dim=128, @@ -34,12 +52,13 @@ def test_topk_flags_initialized_before_buffer_allocation( q_split_threshold=8192, use_self_sampling_topk=use_self_sampling, ) - metadata.kv_cache_manager = SimpleNamespace( + md.kv_cache_manager = SimpleNamespace( tokens_per_block=128, compressed_block_sizes={}, get_cache_indices=Mock(), ) - metadata.is_cuda_graph = False + md.is_cuda_graph = False + metadata = _dsa_without_init(md, params) enabled = enable_heuristic and sm_version >= 100 self_sampling_supported = enabled and dsl_available and sm_version in (100, 103) temporal_supported = enabled and dsl_available and sm_version in (100, 103) @@ -54,11 +73,10 @@ def check_flags(*, capture_graph: bool) -> None: metadata.create_buffers_for_indexer = Mock(side_effect=check_flags) with ( - patch.object(dsa_metadata.TrtllmAttentionMetadata, "__post_init__"), patch.object(dsa_metadata, "IS_CUTLASS_DSL_AVAILABLE", dsl_available), patch.object(dsa_metadata, "get_sm_version", return_value=sm_version), ): - metadata.__post_init__() + construct_extension(metadata) metadata.create_buffers_for_mla_rope_append.assert_called_once_with(capture_graph=False) metadata.create_buffers_for_indexer.assert_called_once_with(capture_graph=False) @@ -68,8 +86,8 @@ def check_flags(*, capture_graph: bool) -> None: @pytest.mark.parametrize("num_local_layers", [1, 3]) def test_temporal_gvr_allocates_real_prior_buffers(num_local_layers: int) -> None: """Allocate actual CUDA buffers and size the zeroed prior by local layers.""" - metadata = object.__new__(dsa_metadata.DSAtrtllmAttentionMetadata) - metadata.sparse_metadata_params = DSAMetadataParams( + md = object.__new__(TrtllmAttentionMetadata) + params = DSAMetadataParams( indexer_max_chunk_size=32, max_sparse_topk=512, index_head_dim=128, @@ -80,30 +98,30 @@ def test_temporal_gvr_allocates_real_prior_buffers(num_local_layers: int) -> Non q_split_threshold=32, use_self_sampling_topk=False, ) - metadata.kv_cache_manager = SimpleNamespace( + md.kv_cache_manager = SimpleNamespace( tokens_per_block=128, compressed_block_sizes={}, get_cache_indices=Mock(), max_blocks_per_seq=2, num_local_layers=num_local_layers, ) - metadata.draft_kv_cache_manager = None - metadata.is_cuda_graph = False - metadata.cuda_graph_buffers = None - metadata.max_num_sequences = 4 - metadata.max_num_tokens = 16 + md.replace_ext(bare_extension(DraftMetadata, md)) + md.is_cuda_graph = False + md.cuda_graph_buffers = None + md.max_num_sequences = 4 + md.max_num_tokens = 16 + md.enable_context_mla_with_cached_kv = False + metadata = _dsa_without_init(md, params) metadata.max_draft_tokens = 3 metadata.num_sms = 16 - metadata.enable_context_mla_with_cached_kv = False # object.__new__ skips __init__, which is where this default is set. metadata._radix_rows_per_sequence = 1 with ( - patch.object(dsa_metadata.TrtllmAttentionMetadata, "__post_init__"), patch.object(metadata, "create_buffers_for_mla_rope_append"), patch.object(dsa_metadata, "IS_CUTLASS_DSL_AVAILABLE", True), patch.object(dsa_metadata, "get_sm_version", return_value=100), ): - metadata.__post_init__() + construct_extension(metadata) assert metadata.enable_gvr_topk assert metadata.needs_gvr_prior diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_mtp_schedule_skip.py b/tests/unittest/_torch/attention/sparse/dsa/test_mtp_schedule_skip.py index dbc3e055b9ef..d97bdfb252c5 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_mtp_schedule_skip.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_mtp_schedule_skip.py @@ -1,15 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""update_for_spec_dec() skips the indexer schedule rebuild only inside an index-sharing draft loop.""" +"""A draft step's before_forward() skips the indexer schedule rebuild only inside an index-sharing draft loop.""" import pytest import torch from utils.util import skip_pre_hopper -from tensorrt_llm._torch.attention.backends.sparse.dsa import ( - DSACacheManager, - DSAtrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig +from tensorrt_llm._torch.attention.backends.sparse.dsa import DSACacheManager, DSAMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.executor import KvCacheConfig @@ -79,7 +78,10 @@ def decode_metadata(): request_ids=request_ids, token_nums=KV_LENS, is_gen=False ) cached = [kv_len - 1 for kv_len in KV_LENS] - metadata = DSAtrtllmAttentionMetadata( + # The draft depth sizes the next_n-dependent buffers. Spec-dec attention + # stays off, as in draft steps >= 1. + metadata = TrtllmAttentionMetadata( + extension_types=(DSAMetadata,), seq_lens=torch.ones(batch_size, dtype=torch.int), request_ids=request_ids, max_num_requests=batch_size, @@ -90,16 +92,9 @@ def decode_metadata(): kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached), mapping=mapping, sparse_metadata_params=sparse_config.to_sparse_metadata_params(), - ) - # The draft depth is set the way the engine sets it; this also sizes the - # next_n-dependent buffers. Spec-dec attention stays off, as in draft steps >= 1. - metadata.update_spec_dec_param( - batch_size=batch_size, - is_spec_decoding_enabled=False, - is_spec_dec_tree=False, - is_spec_dec_dynamic_tree=False, - max_draft_len=MAX_DRAFT_TOKENS, - max_total_draft_tokens=MAX_DRAFT_TOKENS, + spec_decoding=SpecDecodingConfig( + max_draft_len=MAX_DRAFT_TOKENS, max_total_draft_tokens=MAX_DRAFT_TOKENS + ), ) metadata.prepare() torch.cuda.synchronize() @@ -110,7 +105,7 @@ def decode_metadata(): def _schedule_buffers(metadata): - """What on_update_kv_lens(skip_indexer_schedule=True) leaves untouched.""" + """What a skipped indexer schedule rebuild leaves untouched.""" return [ metadata.scheduler_metadata_buffer, metadata.scheduler_metadata_buffer_full_next_n, @@ -129,7 +124,7 @@ def _is_poisoned(metadata): def _kv_lens_buffers(metadata): - """What on_update_kv_lens() refreshes from the KV lengths even when it skips the schedule.""" + """What before_forward() refreshes from the KV lengths even when it skips the schedule.""" num_tokens = metadata.num_tokens num_generations = metadata.num_generations return [ @@ -147,10 +142,11 @@ def _snapshot(buffers): def test_schedule_is_skipped_only_inside_the_draft_loop(decode_metadata): metadata = decode_metadata + dsa = metadata.ext(DSAMetadata) # Outside the draft loop: full rebuild. _poison(metadata) - metadata.update_for_spec_dec() + dsa.refresh_from_draft_lengths() assert not metadata.indexer_schedule_stale assert _is_poisoned(metadata) == [False, False, False] before_advance = _snapshot(_kv_lens_buffers(metadata)) @@ -158,21 +154,30 @@ def test_schedule_is_skipped_only_inside_the_draft_loop(decode_metadata): # Inside the index-sharing draft loop, after a draft step appends a token: the schedule # is left as is and marked stale, but the KV-lens-dependent buffers are still rewritten. metadata.set_in_mtp_draft_loop(True) - metadata.kv_lens_cuda[: metadata.num_generations] += 1 + metadata.adjust_kv_lens(slice(0, metadata.num_generations), 1) _poison(metadata) for buf in _kv_lens_buffers(metadata): buf.fill_(-1) - metadata.update_for_spec_dec() + dsa.refresh_from_draft_lengths() assert metadata.indexer_schedule_stale assert _is_poisoned(metadata) == [True, True, True] in_draft_loop = _snapshot(_kv_lens_buffers(metadata)) - # The next full rebuild (target forward) restores the schedule, clears the marker and - # writes the same slot mappings and gen indptrs, which follow the advanced lengths. - metadata.on_update_kv_lens() + # A full rebuild restores the schedule, clears the marker and writes the same slot + # mappings and gen indptrs, which follow the advanced lengths. + metadata.set_in_mtp_draft_loop(False) + dsa.refresh_from_draft_lengths() assert not metadata.indexer_schedule_stale assert _is_poisoned(metadata) == [False, False, False] full_rebuild = _snapshot(_kv_lens_buffers(metadata)) for skipped, rebuilt, stale in zip(in_draft_loop, full_rebuild, before_advance): assert torch.equal(skipped, rebuilt) assert not torch.equal(skipped, stale) + + # The next step's prepare rebuilds the schedule as well. + metadata.set_in_mtp_draft_loop(True) + dsa.refresh_from_draft_lengths() + assert metadata.indexer_schedule_stale + metadata.set_in_mtp_draft_loop(False) + metadata.prepare() + assert not metadata.indexer_schedule_stale diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py b/tests/unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py index d20bd921a66e..1d8142148473 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py @@ -13,25 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Tests for the token->request map used by DSAtrtllmAttentionMetadata. +Tests for the token->request map used by the DSA metadata extension. 1. build_req_idx_per_token must match the host repeat_interleave build for every layout, so the device and host builders cannot drift. -2. on_update_kv_lens() must rebuild the map after the MTP draft loop rewrites +2. before_forward() must rebuild the map after the MTP draft loop rewrites seq_lens, or every draft token is misattributed to request 0 (https://nvbugs/6513132, https://nvbugs/6513093). -3. on_update_kv_lens() must preserve the base metadata invalidation contract. +3. before_forward() must preserve the base metadata invalidation contract. """ from unittest.mock import Mock import pytest import torch +from _torch.helpers import bare_extension, refresh_from_lengths -from tensorrt_llm._torch.attention.backends.sparse.dsa import ( - DSAtrtllmAttentionMetadata, - build_req_idx_per_token, -) +from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata, build_req_idx_per_token +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata def _host_reference(seq_lens: torch.Tensor) -> torch.Tensor: @@ -65,44 +65,61 @@ def test_matches_host_repeat_interleave(seq_lens, device): assert result.to(torch.int32).tolist() == _host_reference(seq_lens).tolist() -def test_on_update_kv_lens_invalidates_base_mla_state() -> None: - md = object.__new__(DSAtrtllmAttentionMetadata) - md.enable_flash_mla = False - md._mla_scheduler_buffers_valid = True - md._mla_ctx_cu_seqlens_valid = True - md._invalidate_pool_view_cache = Mock() +def _attach_dsa(md: TrtllmAttentionMetadata) -> DSAMetadata: + """A DSA extension built without __init__, bound to ``md``.""" + dsa = object.__new__(DSAMetadata) + dsa.metadata = md + dsa.enable_ragged_verification = False + dsa.ragged_verify_lens = None + md.replace_ext(bare_extension(MTPIndexShareExtension, md)) + md.replace_ext(dsa) + return dsa + + +def test_before_forward_invalidates_base_mla_state() -> None: + md = object.__new__(TrtllmAttentionMetadata) + md.core()._mla_scheduler_buffers_valid = True + md.core()._mla_ctx_cu_seqlens_valid = True md._num_tokens = 0 md._num_generations = 0 md.kv_cache_manager = None md.kv_lens_cuda = torch.empty(0, dtype=torch.int32) - md._compute_kv_lens_row_reorder = Mock() - md.prepare_dense_topk_indices = Mock() + dsa = _attach_dsa(md) + dsa._invalidate_pool_view_cache = Mock() + dsa._compute_kv_lens_row_reorder = Mock() + dsa.prepare_dense_topk_indices = Mock() - md.on_update_kv_lens() + refresh_from_lengths(md) - assert not md._mla_scheduler_buffers_valid - assert not md._mla_ctx_cu_seqlens_valid + assert not md.core()._mla_scheduler_buffers_valid + assert not md.core()._mla_ctx_cu_seqlens_valid -def test_on_update_kv_lens_rebuilds_stale_map() -> None: - """on_update_kv_lens() must replace prepare()'s stale map (fails pre-fix).""" +def test_before_forward_rebuilds_stale_map() -> None: + """before_forward() must replace prepare()'s stale map (fails pre-fix).""" if not torch.cuda.is_available(): pytest.skip("CUDA not available") device = "cuda" max_draft_len = 3 num_requests = 3 - md = object.__new__(DSAtrtllmAttentionMetadata) + md = object.__new__(TrtllmAttentionMetadata) md.kv_cache_manager = None md._num_generations = 0 + dsa = _attach_dsa(md) # __init__ (bypassed by object.__new__) defaults this to False; - # on_update_kv_lens() reads it since #16925. + # before_forward() reads it. md.in_mtp_draft_loop = False - md._group_remap_batched = {} + # Same reason: `_invalidate_pool_view_cache` clears this unconditionally, and + # unlike the sibling test above we exercise the real method rather than a Mock + # (the map rebuild happens downstream of it). A dict, not a Mock, so that a + # future attribute added to that method fails loudly here instead of being + # absorbed. + dsa._group_remap_batched = {} # Stub collaborators unrelated to the map rebuild (test_dsa_indexer.py style). md.kv_lens_cuda = torch.tensor([100, 200, 300], dtype=torch.int32, device=device) - md._compute_kv_lens_row_reorder = Mock() - md.prepare_dense_topk_indices = Mock() + dsa._compute_kv_lens_row_reorder = Mock() + dsa.prepare_dense_topk_indices = Mock() # prepare()-time state: target forward, 1 + max_draft_len tokens/request. target_seq_lens = torch.full((num_requests,), 1 + max_draft_len, dtype=torch.int32) @@ -119,6 +136,6 @@ def test_on_update_kv_lens_rebuilds_stale_map() -> None: md._num_tokens = num_requests assert md.req_idx_per_token[:num_requests].tolist() == [0, 0, 0] - md.on_update_kv_lens() + refresh_from_lengths(md) assert md.req_idx_per_token[:num_requests].tolist() == [0, 1, 2] diff --git a/tests/unittest/_torch/attention/sparse/msa/test_minimax_m3_sparse_attn_decode.py b/tests/unittest/_torch/attention/sparse/msa/test_minimax_m3_sparse_attn_decode.py index e26270df980c..191266fc50c7 100644 --- a/tests/unittest/_torch/attention/sparse/msa/test_minimax_m3_sparse_attn_decode.py +++ b/tests/unittest/_torch/attention/sparse/msa/test_minimax_m3_sparse_attn_decode.py @@ -18,7 +18,9 @@ import pytest import torch +from _torch.helpers import bare_extension +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.fmha.msa_prefill import run_msa_nvfp4_sparse_gqa from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.kernels.msa_utils import ( MSA_REQUIRED_TOPK, @@ -37,8 +39,9 @@ resolve_num_topk_chunks, ) from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaSparseAttentionMetadata, + MiniMaxM3MsaMetadata, ) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._utils import get_sm_version PAGE_SIZE = SPARSE_BLOCK_SIZE @@ -52,6 +55,14 @@ ) +def _msa_metadata() -> TrtllmAttentionMetadata: + """An uninitialized TRTLLM metadata carrying a bare MSA extension.""" + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) + metadata.replace_ext(bare_extension(MiniMaxM3MsaMetadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) + return metadata + + def _flat_page_table(block_table: torch.Tensor, kv_lens_cpu: torch.Tensor) -> torch.Tensor: """Flatten a block table into the per-request page ids fmha_sm100 consumes. @@ -1035,8 +1046,10 @@ def test_nvfp4_sparse_decode_matches_msa_csr_kernel(num_kv_heads, group): kv_lens = torch.tensor(seq_lens, device="cuda", dtype=torch.int32) cu_kv = torch.zeros(batch + 1, device="cuda", dtype=torch.int32) torch.cumsum(kv_lens, 0, out=cu_kv[1:]) - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) - metadata.__dict__.update( + metadata = _msa_metadata() + metadata._num_contexts = 2 + metadata._num_generations = batch - 2 + metadata.ext(MiniMaxM3MsaMetadata).__dict__.update( _msa_live_batch=batch, msa_cu_q_lens=torch.arange(batch + 1, device="cuda", dtype=torch.int32), msa_cu_kv_lens=cu_kv, @@ -1046,14 +1059,12 @@ def test_nvfp4_sparse_decode_matches_msa_csr_kernel(num_kv_heads, group): _msa_total_k_rows=sum((s + PAGE_SIZE - 1) // PAGE_SIZE for s in seq_lens), msa_block_table=case.block_table, msa_seq_lens_cuda=case.seq_lens, - _num_contexts=2, _msa_context_prefix_bounds=( 1, max(seq_lens[:2]), sum(seq_lens[:2]), sum((s + PAGE_SIZE - 1) // PAGE_SIZE for s in seq_lens[:2]), ), - _num_generations=batch - 2, ) triton_out = _run_nvfp4(case) @@ -1137,7 +1148,7 @@ def test_msa_nvfp4_calibrated_staging_and_scale_replay( ) k_global = torch.tensor([2.0 / 2688.0], device="cuda", dtype=torch.float32) v_global = torch.tensor([1.0 / 2688.0], device="cuda", dtype=torch.float32) - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) + metadata = _msa_metadata() metadata._msa_live_batch = 1 metadata.msa_cu_q_lens = torch.tensor([0, 1], device="cuda", dtype=torch.int32) metadata.msa_cu_kv_lens = torch.tensor([0, kv_len], device="cuda", dtype=torch.int32) diff --git a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py index 240bd68ec98d..900a85bede99 100644 --- a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py +++ b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py @@ -17,7 +17,10 @@ import pytest import torch +from _torch.helpers import bare_extension +from tensorrt_llm._torch.attention.backends.features import DraftMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import ( MiniMaxM3KVCacheManagerV2, MiniMaxM3MsaSparseAttention, @@ -37,31 +40,35 @@ minimax_m3_sparse_attn_decode, ) from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaSparseAttentionMetadata, + MiniMaxM3MsaMetadata, MsaDecodeSpan, ) from tensorrt_llm._torch.attention.backends.sparse.registry import _resolve_minimax_m3_backend_cls +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.runtime_hooks import MsaRuntimePolicy from tensorrt_llm.bindings import DataType from tensorrt_llm.llmapi.llm_args import MiniMaxM3SparseAttentionConfig +def _msa_metadata() -> TrtllmAttentionMetadata: + """An uninitialized TRTLLM metadata carrying a bare MSA extension.""" + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) + metadata.replace_ext(bare_extension(MiniMaxM3MsaMetadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) + return metadata + + @pytest.mark.cpu_only -@pytest.mark.parametrize( - "tail_in_forward", [True, False], ids=["tail_in_forward", "tail_in_prepare"] -) @pytest.mark.parametrize("empty_state", ["empty_batch", "no_requests", "no_lengths"]) def test_msa_metadata_clears_padded_cache_slot_tail( - monkeypatch: pytest.MonkeyPatch, empty_state: str, tail_in_forward: bool + monkeypatch: pytest.MonkeyPatch, empty_state: str ) -> None: """A smaller replay must not reuse the previous step's live cache slots.""" from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import msa_backend monkeypatch.setattr(msa_backend, "maybe_pin_memory", lambda tensor: tensor) - monkeypatch.setattr(msa_backend, "TAIL_FILL_IN_FORWARD", tail_in_forward) - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata._msa_buffers_ready = True metadata.request_ids = [0] metadata.kv_cache_manager = SimpleNamespace( @@ -85,8 +92,8 @@ def test_msa_metadata_clears_padded_cache_slot_tail( # live count off it. metadata._num_tokens = count metadata._build_msa_fields() - # TAIL_FILL_IN_FORWARD: the forward (on_update_kv_lens) fills the tail before anything reads it. - metadata._fill_tail_in_forward() + # TAIL_FILL_IN_FORWARD: the forward (msa_write_slots) fills the tail before anything reads it. + metadata.msa_write_slots() assert metadata.msa_out_cache_loc.tolist() == list(range(12, 12 + count)) + [-1] * ( 4 - count ) @@ -408,7 +415,7 @@ def resolve_index_view( ) for _ in range(2) ] - metadata = [object.__new__(MiniMaxM3MsaSparseAttentionMetadata) for _ in managers] + metadata = [_msa_metadata() for _ in managers] for manager, meta in zip(managers, metadata): meta.kv_cache_manager = manager for layer in (0, 5, 60): @@ -421,7 +428,7 @@ def resolve_index_view( if implementation == "msa": assert meta.msa_idx_k_cache(3) is cache - def forward(meta: MiniMaxM3MsaSparseAttentionMetadata, value: torch.Tensor) -> torch.Tensor: + def forward(meta: TrtllmAttentionMetadata, value: torch.Tensor) -> torch.Tensor: get_cache = ( meta.msa_idx_k_cache if implementation == "msa" @@ -460,8 +467,7 @@ def test_cache_manager_rejects_non_positive_sparse_index_dim(sparse_index_dim: i def test_msa_metadata_rejects_undersized_max_score_buffer(): - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() # Flat backing store sized for 4 heads * 8 k-tiles * 2 batch = 64 elements, # too small for the plan's required 4 * 16 * 2 = 128. One query token per # decode row, so the token bound equals the batch here. @@ -496,8 +502,7 @@ def get_buffer(self, tensor_shape, dtype, cache_name, capture_graph): def _buffer_metadata(**manager_fields): """Metadata ready for _create_msa_buffers, under capture.""" - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata.kv_cache_manager = SimpleNamespace( max_blocks_per_seq=MAX_BLOCKS_PER_SEQ, tokens_per_block=128, @@ -596,6 +601,7 @@ def test_msa_buffers_reserve_cumulative_lengths_for_csr_prefill( from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import msa_backend metadata = _buffer_metadata(dtype=kv_dtype, sparse_layer_ids=[], layer_offsets={}) + msa = metadata.ext(MiniMaxM3MsaMetadata) reserved = {} def get_empty(buffers, shape, **kwargs): @@ -611,10 +617,10 @@ def get_empty(buffers, shape, **kwargs): assert reserved["msa_qo_lens_cuda"] == (MAX_NUM_SEQUENCES,) assert reserved["msa_cu_q_lens"] == (MAX_NUM_SEQUENCES + 1,) assert reserved["msa_cu_kv_lens"] == (MAX_NUM_SEQUENCES + 1,) - assert all(getattr(metadata, name) is not None for name in names) + assert all(getattr(msa, name) is not None for name in names) else: assert not set(names) & set(reserved) - assert all(getattr(metadata, name) is None for name in names) + assert all(getattr(msa, name) is None for name in names) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -654,8 +660,7 @@ def test_msa_subpage_rows_slice_the_generation_span(): """A mixed step hands the dense kernel only the span's rows, so its sub-page table has to be sliced the same way the slot table is. The factor travels with it so the kernel can tell a stale staging from its own geometry.""" - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata.msa_subpage_block_table = torch.arange(4 * 2 * 3, dtype=torch.int32).reshape(4, 2, 3) metadata._msa_subpages_per_slot = 9 @@ -674,8 +679,7 @@ def test_msa_proxy_max_score_view_is_contiguous_over_stable_store(): [num_index_heads, plan_max_k_tiles, num_tokens] shape the kernel writes, backed by a stable store so its data_ptr survives CUDA graph replay. """ - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() # Worst-case store: 4 heads * 16 k-tiles * 8 batch. num_index_heads, worst_k, max_batch = 4, 16, 8 metadata.msa_max_score = torch.zeros(num_index_heads * worst_k * max_batch) @@ -803,8 +807,7 @@ def test_per_token_valid_blocks_multi_token_decode(): def test_msa_index_k_uses_hnd_cache_view_and_writer(): - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() num_pages, coalescing_scale, page_size, head_dim = 2, 7, 8, 16 pool = torch.zeros( num_pages, @@ -931,7 +934,7 @@ class FakeMetadata: # indexer takes the proxy path and needs no decode span. msa_decode_span = None msa_prefill_proxy_plan = (False, 0, 2, {}, None) - _msa_prefill_identity = None + msa_prefill_identity = None msa_prefill_n_valid_blocks = torch.ones(2, dtype=torch.int32, device="cuda") msa_prefill_proxy_kv_indices = torch.arange(2, dtype=torch.int32, device="cuda") msa_qo_lens_cpu = torch.ones(2, dtype=torch.int32) @@ -945,6 +948,10 @@ def __init__(self, dtype: torch.dtype) -> None: self.cache = backing[::7] self.msa_out_cache_loc = torch.tensor([0, 128], dtype=torch.int32, device="cuda") + def ext(self, cls: type) -> "FakeMetadata": + # The fake serves as its own MSA extension. + return self + def msa_write_idx_k(self, layer_idx: int, idx_k: torch.Tensor) -> None: from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.common import ( write_kv_slots, @@ -1049,7 +1056,7 @@ def select_blocks(self, *args: object, **kwargs: object) -> torch.Tensor: return torch.zeros(num_tokens, 1, 16, dtype=torch.int32) class FakeMetadata: - _msa_prefill_identity = None + msa_prefill_identity = None msa_prefill_n_valid_blocks = torch.ones(num_tokens, dtype=torch.int32) msa_n_valid_blocks = torch.ones(num_tokens, dtype=torch.int32) @@ -1080,6 +1087,10 @@ def __init__(self) -> None: dtype=torch.bfloat16, ) + def ext(self, cls: type) -> "FakeMetadata": + # The fake serves as its own MSA extension. + return self + def msa_write_idx_k(self, layer_idx: int, idx_k: torch.Tensor) -> None: del layer_idx self.idx_k_cache = idx_k @@ -1300,8 +1311,7 @@ def test_lazily_allocated_scratch_publishes_the_bound_it_used(monkeypatch): monkeypatch.setattr(msa_backend, "require_msa_module", lambda: None) monkeypatch.setattr(msa_backend, "_worst_case_proxy_max_k_tiles", lambda *a, **kw: 32) - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata.kv_cache_manager = SimpleNamespace() metadata.cuda_graph_buffers = None metadata.max_num_sequences = 2 @@ -1328,8 +1338,7 @@ def _span_metadata(*, num_contexts=0, qo_lens=(1, 1), kv_lens=(9, 11), is_cuda_g enough to drive the real msa_*_cpu length properties; no cache pool is needed. """ - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata.mapping = None # Assigned behind the seq_lens property, whose setter would stage a device # copy nothing here reads; num_seqs derives from it. The num_contexts @@ -1625,6 +1634,8 @@ def test_decode_fmha_runs_the_phase_its_span_describes(): msa_block_table=torch.zeros(4, 4, dtype=torch.int32), msa_seq_lens_cuda=torch.zeros(4, dtype=torch.int32), ) + # The stub serves as its own MSA extension. + metadata.ext = lambda cls: metadata params = _generation_params(attention, metadata, seq_offset=2, input_seq_length=1) dispatched = [] decode._run_dense = lambda params, block_table, seq_lens: dispatched.append( @@ -1656,6 +1667,9 @@ def test_each_phase_writes_the_cache_slots_of_the_rows_it_attends(): kv_cache_manager=SimpleNamespace(get_buffers=lambda layer_idx, kv_layout=None: buffers), msa_out_cache_loc=torch.tensor([1, 2, page_size + 3], dtype=torch.int32), ) + metadata.msa_write_slots = lambda num_tokens=None: metadata.msa_out_cache_loc + # The stub serves as its own MSA extension. + metadata.ext = lambda cls: metadata attention = SimpleNamespace(layer_idx=0) k = torch.arange(1, 3 * head_dim + 1, dtype=torch.float32).reshape(3, head_dim) v = -k @@ -1845,6 +1859,8 @@ def test_mixed_batch_generation_span_matches_the_whole_batch_msa_path(): ) total_q = int(q.shape[0]) metadata = SimpleNamespace(**fields) + # The namespace serves as its own MSA extension. + metadata.ext = lambda cls: metadata num_heads = attention.num_heads sm_scale = head_dim**-0.5 @@ -1907,8 +1923,7 @@ def test_msa_scratch_sizing_covers_spec_verify_tokens(): proxy scratch must be sized by tokens, not by batch. The draft length comes from the KV cache manager, which knows the speculative config at build time. """ - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() # 2 sequences, 4 tokens each (draft_len=3): 8 decode tokens per step. metadata.kv_cache_manager = SimpleNamespace(max_total_draft_tokens=3) metadata.max_num_sequences = 2 @@ -1926,21 +1941,15 @@ def test_msa_scratch_sizing_covers_spec_verify_tokens(): ) -def _kv_lens_update_metadata( - monkeypatch, *, qo_lens, kv_staged, kv_corrected, page_size, prefill_counts=False -): - """Metadata with just enough staged state for on_update_kv_lens: requests with - several query tokens each, optimistic kv_lens from prepare(), then the - corrected kv_lens. Everything lives on the host. `prefill_counts` stages the +def _kv_lens_update_metadata(*, qo_lens, kv_staged, kv_corrected, page_size, prefill_counts=False): + """Metadata with just enough staged state for the device KV-length correction + prepare() applies with kv_lens_offsets: requests with several query tokens + each, optimistic kv_lens from prepare(), then the corrected kv_lens. + Everything lives on the host. `prefill_counts` stages the per-step valid-block buffer a mixed step uses in place of the graph-stable one. """ - from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import msa_backend - - # The base hook only touches MLA caches this metadata never builds. - monkeypatch.setattr(msa_backend.TrtllmAttentionMetadata, "on_update_kv_lens", lambda self: None) - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() qo = torch.tensor(qo_lens, dtype=torch.int32) batch = int(qo.shape[0]) total_q = int(qo.sum()) @@ -1977,13 +1986,13 @@ def _kv_lens_update_metadata( @pytest.mark.parametrize("nvfp4", [False, True]) -def test_on_update_kv_lens_rederives_lengths_slots_and_counts(monkeypatch, nvfp4): +def test_on_update_kv_lens_rederives_lengths_slots_and_counts(nvfp4): """Request 0 loses one rejected draft token (staged 9 -> corrected 8), request 1 is unchanged. The per-request length the decode kernels read, the write slots and the per-token valid-block counts must all follow. """ metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(2, 3), kv_staged=(9, 12), kv_corrected=(8, 12), page_size=4 + qo_lens=(2, 3), kv_staged=(9, 12), kv_corrected=(8, 12), page_size=4 ) if nvfp4: @@ -1992,8 +2001,8 @@ def test_on_update_kv_lens_rederives_lengths_slots_and_counts(monkeypatch, nvfp4 metadata.msa_cu_kv_lens = torch.tensor([0, 9, 21, -1], dtype=torch.int32) metadata.msa_cu_q_lens = torch.tensor([0, 2, 5], dtype=torch.int32) - metadata.on_update_kv_lens() - metadata.on_update_kv_lens() # Corrections must be idempotent. + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() # Corrections must be idempotent. if nvfp4: assert metadata.msa_cu_kv_lens.tolist() == [0, 8, 20, -1] assert metadata.msa_cu_q_lens.tolist() == [0, 2, 5] @@ -2010,29 +2019,28 @@ def test_on_update_kv_lens_rederives_lengths_slots_and_counts(monkeypatch, nvfp4 assert metadata.msa_n_valid_blocks[5:].tolist() == [0, 0, 0] -def test_on_update_kv_lens_clamps_to_the_staged_lens(monkeypatch): +def test_on_update_kv_lens_clamps_to_the_staged_lens(): """A correction can only shrink lengths; anything above the staged value is clamped, so the staged page table stays valid. """ metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(1, 1), kv_staged=(5, 7), kv_corrected=(9, 7), page_size=4 + qo_lens=(1, 1), kv_staged=(5, 7), kv_corrected=(9, 7), page_size=4 ) - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [5, 7] assert metadata.msa_out_cache_loc[:2].tolist() == [4, 106] assert metadata.msa_n_valid_blocks[:2].tolist() == [2, 2] -def test_on_update_kv_lens_patches_a_mixed_steps_per_step_counts(monkeypatch): +def test_on_update_kv_lens_patches_a_mixed_steps_per_step_counts(): """A mixed step stages its valid-block counts in the per-step prefill buffer rather than the graph-stable one, and the correction must land there. The context row (row 0) is untouched by the correction; the generation row (row 1) loses one token. """ metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(5, 3), kv_staged=(5, 12), kv_corrected=(5, 11), @@ -2040,7 +2048,7 @@ def test_on_update_kv_lens_patches_a_mixed_steps_per_step_counts(monkeypatch): prefill_counts=True, ) - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [5, 11] # Context row: positions 0..4 -> slots 0..4. Generation row attends 11 @@ -2051,17 +2059,17 @@ def test_on_update_kv_lens_patches_a_mixed_steps_per_step_counts(monkeypatch): assert metadata.msa_n_valid_blocks.tolist() == [0] * 11 -def test_on_update_kv_lens_is_a_noop_without_speculative_decoding(monkeypatch): +def test_on_update_kv_lens_is_a_noop_without_speculative_decoding(): """Without speculative decoding the patch and its staging are skipped. The gate also sees max_total_draft_tokens, set by update_spec_dec_param, which covers draft_len=1 under trtllm-gen (no extra KV tokens, linear-tree spec decoding reported disabled). """ metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(1, 1), kv_staged=(4, 6), kv_corrected=(3, 5), page_size=4 + qo_lens=(1, 1), kv_staged=(4, 6), kv_corrected=(3, 5), page_size=4 ) metadata.max_total_draft_tokens = None - metadata.draft_kv_cache_manager = None + metadata.replace_ext(bare_extension(DraftMetadata, metadata)) metadata.is_spec_decoding_enabled = False metadata.kv_cache_params = SimpleNamespace(num_extra_kv_tokens=0) metadata.runtime_features = None @@ -2074,7 +2082,7 @@ def test_on_update_kv_lens_is_a_noop_without_speculative_decoding(monkeypatch): metadata._msa_kv_lens_dynamic = False - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [4, 6] assert metadata.msa_out_cache_loc.tolist() == [-1] * 5 @@ -2124,9 +2132,9 @@ def __call__( return self._taken -def test_default_msa_policy_has_no_kv_update(monkeypatch): +def test_default_msa_policy_has_no_kv_update(): metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(1,), kv_staged=(5,), kv_corrected=(5,), page_size=4 + qo_lens=(1,), kv_staged=(5,), kv_corrected=(5,), page_size=4 ) assert metadata.msa_policy == MsaRuntimePolicy() @@ -2134,9 +2142,8 @@ def test_default_msa_policy_has_no_kv_update(monkeypatch): @pytest.mark.parametrize("prefill_counts", [False, True]) -def test_a_taken_kv_update_replaces_the_rederivation(monkeypatch, prefill_counts): +def test_a_taken_kv_update_replaces_the_rederivation(prefill_counts): metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(2, 3), kv_staged=(9, 12), kv_corrected=(8, 12), @@ -2146,7 +2153,7 @@ def test_a_taken_kv_update_replaces_the_rederivation(monkeypatch, prefill_counts kv_update = _RecordingKvUpdate(taken=True) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() # The hook owns the update: the metadata's own re-derivation does not run. assert metadata.msa_seq_lens_cuda.tolist() == [9, 12] @@ -2169,14 +2176,14 @@ def test_a_taken_kv_update_replaces_the_rederivation(monkeypatch, prefill_counts assert call["n_valid_blocks"].numel() == 5 -def test_a_declined_kv_update_falls_back_to_the_rederivation(monkeypatch): +def test_a_declined_kv_update_falls_back_to_the_rederivation(): metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(2, 3), kv_staged=(9, 12), kv_corrected=(8, 12), page_size=4 + qo_lens=(2, 3), kv_staged=(9, 12), kv_corrected=(8, 12), page_size=4 ) kv_update = _RecordingKvUpdate(taken=False) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() assert len(kv_update.calls) == 1 assert metadata.msa_seq_lens_cuda.tolist() == [8, 12] @@ -2184,15 +2191,15 @@ def test_a_declined_kv_update_falls_back_to_the_rederivation(monkeypatch): assert metadata.msa_n_valid_blocks[:5].tolist() == [2, 2, 3, 3, 3] -def test_kv_update_is_not_called_without_dynamic_kv_lens(monkeypatch): +def test_kv_update_is_not_called_without_dynamic_kv_lens(): metadata = _kv_lens_update_metadata( - monkeypatch, qo_lens=(1, 1), kv_staged=(4, 6), kv_corrected=(3, 5), page_size=4 + qo_lens=(1, 1), kv_staged=(4, 6), kv_corrected=(3, 5), page_size=4 ) metadata._msa_kv_lens_dynamic = False kv_update = _RecordingKvUpdate(taken=True) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.on_update_kv_lens() + metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() assert kv_update.calls == [] diff --git a/tests/unittest/_torch/attention/sparse/msa/test_msa_runtime_policy.py b/tests/unittest/_torch/attention/sparse/msa/test_msa_runtime_policy.py index f76722ec6d1b..e825b66d4e08 100644 --- a/tests/unittest/_torch/attention/sparse/msa/test_msa_runtime_policy.py +++ b/tests/unittest/_torch/attention/sparse/msa/test_msa_runtime_policy.py @@ -12,6 +12,7 @@ import numpy as np import pytest import torch +from _torch.helpers import bare_extension from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import ( MiniMaxM3KVCacheManagerV2, @@ -27,9 +28,10 @@ write_subpage_block_table, ) from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaSparseAttentionMetadata, + MiniMaxM3MsaMetadata, _fill_unused_slots, ) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor import h2d_cache from tensorrt_llm._torch.pyexecutor.h2d_cache import H2DSkipCache from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 @@ -81,19 +83,28 @@ def no_index_k_buffer(self, layer_idx, **kwargs): # --------------------------------------------------------------------------- +def _bare_msa() -> MiniMaxM3MsaMetadata: + """The MSA extension of an uninitialized TRTLLM metadata, with no buffers.""" + metadata = TrtllmAttentionMetadata.__new__(TrtllmAttentionMetadata) + msa = bare_extension(MiniMaxM3MsaMetadata, metadata) + metadata.replace_ext(msa) + return msa + + def _short_table_metadata( policy: MsaRuntimePolicy, manager: MiniMaxM3KVCacheManagerV2, num_contexts: int = 0 -) -> MiniMaxM3MsaSparseAttentionMetadata: - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) - metadata.msa_policy = policy - metadata._num_contexts = num_contexts - metadata.kv_cache_manager = manager - metadata.max_num_sequences = 4 - metadata.max_num_tokens = 16 - metadata.cuda_graph_buffers = None - metadata.is_cuda_graph = False - metadata.mapping = Mapping() - return metadata +) -> MiniMaxM3MsaMetadata: + msa = _bare_msa() + msa.msa_policy = policy + core = msa.metadata.core() + core._num_contexts = num_contexts + core.kv_cache_manager = manager + core.max_num_sequences = 4 + core.max_num_tokens = 16 + core.cuda_graph_buffers = None + core.is_cuda_graph = False + core.mapping = Mapping() + return msa def _identity(n_valid: list[int]) -> torch.Tensor: @@ -103,7 +114,7 @@ def _identity(n_valid: list[int]) -> torch.Tensor: return table.unsqueeze(0).expand(NUM_KV_HEADS, len(n_valid), MSA_REQUIRED_TOPK) -def _staged(metadata: MiniMaxM3MsaSparseAttentionMetadata, total_q: int) -> torch.Tensor: +def _staged(metadata: MiniMaxM3MsaMetadata, total_q: int) -> torch.Tensor: torch.cuda.synchronize() size = NUM_KV_HEADS * total_q * MSA_REQUIRED_TOPK return metadata.msa_short_table[:size].view(NUM_KV_HEADS, total_q, MSA_REQUIRED_TOPK).cpu() @@ -143,7 +154,7 @@ def test_ineligible_steps_do_not_use_the_identity_table(manager, n_valid, num_co metadata._stage_msa_short_table(torch.tensor([3], dtype=torch.int32), PARAMS) assert metadata._msa_short_ok - metadata._num_contexts = num_contexts + metadata.metadata._num_contexts = num_contexts metadata._stage_msa_short_table(torch.tensor(n_valid, dtype=torch.int32), PARAMS) assert not metadata._msa_short_ok @@ -470,10 +481,8 @@ def test_block_ids_per_seq_matches_the_cache_index_rows(seed): # --------------------------------------------------------------------------- -def _subpage_metadata( - rows: int, max_blocks: int, factor: int -) -> MiniMaxM3MsaSparseAttentionMetadata: - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) +def _subpage_metadata(rows: int, max_blocks: int, factor: int) -> MiniMaxM3MsaMetadata: + metadata = _bare_msa() metadata._msa_subpage_rows = rows # as prepare() stages them metadata.msa_block_table = torch.randint( 0, 1 << 20, (rows + 2, max_blocks), dtype=torch.int32, device="cuda" @@ -518,7 +527,7 @@ def test_each_forward_expands_the_current_block_table(monkeypatch): metadata = _subpage_metadata(rows=2, max_blocks=16, factor=4) metadata.msa_subpage_rows(0, 2) metadata.msa_block_table[:2, :5] += 11 - # The next forward (on_update_kv_lens opens it). + # The next forward: its prepare (or a draft's before_forward) resets the flag. metadata._msa_subpage_expanded = False table, _ = metadata.msa_subpage_rows(0, 2) @@ -552,7 +561,7 @@ def test_a_captured_forward_expands_the_table_at_every_replay(monkeypatch): def test_nothing_is_expanded_without_a_prepared_step(): - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) + metadata = _bare_msa() metadata.msa_subpage_block_table = torch.arange(12, dtype=torch.int32).reshape(2, 2, 3) metadata._msa_subpages_per_slot = 2 @@ -566,25 +575,26 @@ def test_nothing_is_expanded_without_a_prepared_step(): # --------------------------------------------------------------------------- -def _tail_metadata(cap: int, start) -> MiniMaxM3MsaSparseAttentionMetadata: - metadata = object.__new__(MiniMaxM3MsaSparseAttentionMetadata) +def _tail_metadata(cap: int, start) -> MiniMaxM3MsaMetadata: + metadata = _bare_msa() metadata.msa_out_cache_loc = torch.arange(100, 100 + cap, dtype=torch.int32, device="cuda") metadata._msa_tail_start = start + metadata._msa_tail_filled = False return metadata @requires_cuda def test_the_forward_fills_only_past_the_live_slots(): metadata = _tail_metadata(10, 4) - metadata._fill_tail_in_forward() + metadata.msa_write_slots() assert metadata.msa_out_cache_loc.tolist() == [100, 101, 102, 103] + [-1] * 6 untouched = _tail_metadata(6, None) - untouched._fill_tail_in_forward() + untouched.msa_write_slots() assert untouched.msa_out_cache_loc.tolist() == list(range(100, 106)) full = _tail_metadata(6, 6) - full._fill_tail_in_forward() + full.msa_write_slots() assert full.msa_out_cache_loc.tolist() == list(range(100, 106)) @@ -598,7 +608,9 @@ def test_a_captured_forward_refills_its_tail_at_every_replay(): stream.wait_stream(torch.cuda.current_stream()) graph = torch.cuda.CUDAGraph() with torch.cuda.stream(stream), torch.cuda.graph(graph, stream=stream): - metadata._fill_tail_in_forward() + metadata._msa_tail_filled = False + metadata.msa_write_slots() + metadata.msa_write_slots(5) torch.cuda.current_stream().wait_stream(stream) for step in range(3): diff --git a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py index c2b78b4996e0..d4c576ff86d8 100644 --- a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py +++ b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py @@ -19,7 +19,10 @@ import pytest import torch +from utils.metadata_contract import check_copy_reproduces_creation_config +from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import ( QSA_INDEX_POSITION, QSA_SIDE_PLANES, @@ -318,3 +321,23 @@ def test_transfer_views_do_not_depend_on_the_kv_head_split(dtype: DataType) -> N assert roles == {str(Role.INDEX_KEY), str(QSA_INDEX_POSITION)} else: assert roles == {str(QSA_SIDE_PLANES)} + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_qsa_metadata_copies_reproduce_creation_config(): + """A constructed copy of a QSA metadata over the trunk manager reports its + creation configuration.""" + manager = _manager(DataType.BF16, draft=False) + try: + metadata = TrtllmAttentionMetadata( + max_num_requests=4, + max_num_tokens=8, + kv_cache_manager=manager, + extension_types=(QSAMetadata,), + sparse_metadata_params=QSASparseAttentionConfig().to_sparse_metadata_params( + pretrained_config=_CHECKPOINT + ), + ) + check_copy_reproduces_creation_config(metadata) + finally: + manager.shutdown() diff --git a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_sparse.py b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_sparse.py index ec6f97521186..e2be97b9212d 100644 --- a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_sparse.py +++ b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_sparse.py @@ -6,7 +6,9 @@ import pytest import torch +from _torch.helpers import bare_extension, construct_extension, refresh_from_lengths, stub_extension +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension from tensorrt_llm._torch.attention.backends.sparse.qsa import ( QSASparseMetadataParams, QSASparseParams, @@ -30,7 +32,10 @@ triton_qsa_prefill_compress, triton_qsa_unscale_block_table, ) -from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import ( + QSAAttentionMetadata, + QSAMetadata, +) from tensorrt_llm._torch.attention.backends.sparse.qsa.module import ( _capture_mtp_shared_topk, _store_paged_kv_reference, @@ -45,10 +50,6 @@ QSA_SIDE_PLANES, Qwen4ExpHybridCacheManager, ) -from tensorrt_llm._torch.speculative.interface import ( - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, -) from tensorrt_llm.runtime.kv_cache_manager_v2 import PageIndexMode @@ -726,12 +727,20 @@ def test_qsa_generation_token_mapping_rejects_outputs_on_another_device() -> Non triton_qsa_generation_token_mapping(**arguments) +def _bare_qsa_metadata() -> QSAAttentionMetadata: + """A QSA metadata made without ``__init__`` whose QSA state the test sets.""" + metadata = object.__new__(QSAAttentionMetadata) + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + metadata.replace_ext(stub_extension(QSAMetadata, metadata)) + return metadata + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") def test_qsa_spec_dec_remaps_rows_after_the_draft_loop_packs_one_row_per_request() -> None: """The first draft step repacks the batch, so the row mapping must follow it.""" num_seqs, rows_per_seq = 2, 2 num_tokens = num_seqs * rows_per_seq - metadata = object.__new__(QSAAttentionMetadata) + metadata = _bare_qsa_metadata() metadata._seq_lens = torch.full((num_seqs,), rows_per_seq, dtype=torch.int32) metadata._seq_lens_cuda = metadata._seq_lens.cuda() metadata._num_contexts = 0 @@ -739,24 +748,23 @@ def test_qsa_spec_dec_remaps_rows_after_the_draft_loop_packs_one_row_per_request metadata.kv_lens_cuda_runtime = torch.tensor([40, 56], dtype=torch.int32, device="cuda") metadata.sparse_metadata_params = SimpleNamespace(compress_ratio=4) metadata.qsa_has_local_layers = True - metadata.enable_flash_mla = False - metadata._mla_scheduler_buffers_valid = True - metadata._mla_ctx_cu_seqlens_valid = True + metadata.core()._mla_scheduler_buffers_valid = True + metadata.core()._mla_ctx_cu_seqlens_valid = True metadata.qsa_req_idx_per_token = torch.zeros(num_tokens, dtype=torch.int32, device="cuda") metadata.qsa_logical_positions = torch.zeros(num_tokens, dtype=torch.int64, device="cuda") metadata.qsa_sequence_lengths = torch.zeros(num_tokens, dtype=torch.int32, device="cuda") metadata.qsa_visible_blocks = torch.zeros(num_tokens, dtype=torch.int32, device="cuda") - metadata._qsa_cu_seq_lens = torch.zeros(num_seqs + 1, dtype=torch.int64, device="cuda") - metadata._qsa_token_arange = torch.arange(num_tokens, dtype=torch.int64, device="cuda") + metadata._cu_seq_lens = torch.zeros(num_seqs + 1, dtype=torch.int64, device="cuda") + metadata._token_arange = torch.arange(num_tokens, dtype=torch.int64, device="cuda") - metadata._refresh_qsa_token_mapping() + metadata._refresh_token_mapping() assert metadata.qsa_req_idx_per_token.tolist() == [0, 0, 1, 1] # What the draft loop does to the metadata between its first two steps. metadata._seq_lens.fill_(1) metadata._seq_lens_cuda.fill_(1) metadata._num_tokens = num_seqs - metadata.update_for_spec_dec() + refresh_from_lengths(metadata) assert metadata.qsa_req_idx_per_token[:num_seqs].tolist() == [0, 1] torch.testing.assert_close( @@ -779,7 +787,7 @@ def test_qsa_metadata_maps_generation_rows_in_one_launch_up_to_the_max( request count and with the general mapping past it; all four buffers match either way.""" seq_lens = [1 + i % 4 for i in range(num_seqs)] num_tokens = sum(seq_lens) - metadata = object.__new__(QSAAttentionMetadata) + metadata = _bare_qsa_metadata() metadata._seq_lens = torch.tensor(seq_lens, dtype=torch.int32) metadata._seq_lens_cuda = metadata._seq_lens.cuda() metadata._num_contexts = 0 @@ -800,8 +808,8 @@ def test_qsa_metadata_maps_generation_rows_in_one_launch_up_to_the_max( metadata.qsa_sequence_lengths, metadata.qsa_visible_blocks, ) = buffers - metadata._qsa_cu_seq_lens = torch.zeros(num_seqs + 1, dtype=torch.int64, device="cuda") - metadata._qsa_token_arange = torch.arange(num_tokens, dtype=torch.int64, device="cuda") + metadata._cu_seq_lens = torch.zeros(num_seqs + 1, dtype=torch.int64, device="cuda") + metadata._token_arange = torch.arange(num_tokens, dtype=torch.int64, device="cuda") launches = [] def recording_mapping(**kwargs) -> None: @@ -809,12 +817,12 @@ def recording_mapping(**kwargs) -> None: triton_qsa_generation_token_mapping(**kwargs) monkeypatch.setattr( - sys.modules[QSAAttentionMetadata.__module__], + sys.modules[QSAMetadata.__module__], "triton_qsa_generation_token_mapping", recording_mapping, ) - metadata._refresh_qsa_token_mapping() + metadata._refresh_token_mapping() assert launches == ([num_tokens] if one_launch else []) expected = _general_token_mapping( @@ -1856,11 +1864,11 @@ def test_qsa_speculative_snapshot_state_uses_host_lengths( num_contexts: int, expected: bool, ) -> None: - metadata = object.__new__(QSAAttentionMetadata) + metadata = _bare_qsa_metadata() metadata._seq_lens = torch.tensor(sequence_lengths, dtype=torch.int32) metadata._num_contexts = num_contexts - metadata._refresh_qsa_speculative_snapshot_state() + metadata._refresh_speculative_snapshot_state() assert metadata.qsa_needs_speculative_snapshot is expected @@ -1968,198 +1976,115 @@ def _qsa_layout_manager( _DRAFT_LAYOUT = dict(sparse_layer_ids=[48], pool_id=0, scale=4, num_pools=1, max_blocks_per_seq=4) -def _qsa_metadata_with_draft( - monkeypatch, - *, - mtp_index_share: bool = False, - draft_layout: dict | None = _DRAFT_LAYOUT, - draft_manager=None, -): - """QSA metadata over a target manager and, unless `draft_layout` is None, a separate - draft manager (`draft_manager` replaces the one `draft_layout` describes).""" - monkeypatch.setattr(TrtllmAttentionMetadata, "__post_init__", lambda self: None) - metadata = object.__new__(QSAAttentionMetadata) - metadata.kv_cache_manager = _qsa_layout_manager(**_TARGET_LAYOUT) - if draft_manager is None and draft_layout is not None: - draft_manager = _qsa_layout_manager(**draft_layout) - metadata.draft_kv_cache_manager = draft_manager - metadata.sparse_metadata_params = QSASparseMetadataParams( - token_topk=8, compress_ratio=4, mtp_index_share=mtp_index_share - ) +def _qsa_metadata_over(layout: dict, *, mtp_index_share: bool = False): + """QSA metadata over a manager with ``layout``.""" + metadata = object.__new__(TrtllmAttentionMetadata) + metadata.kv_cache_manager = _qsa_layout_manager(**layout) metadata.max_num_sequences = 4 metadata.max_num_tokens = 16 - metadata.draft_replay_swapped_attrs = {} - metadata.__post_init__() + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + qsa = stub_extension(QSAMetadata, metadata) + qsa.sparse_metadata_params = QSASparseMetadataParams( + token_topk=8, compress_ratio=4, mtp_index_share=mtp_index_share + ) + metadata.replace_ext(qsa) + construct_extension(qsa) return metadata -def _set_qsa_offsets(metadata, num_seqs: int, generation: int): - """Write fresh target and draft attention offsets; return the raw slots each - manager's table must decode to.""" - slots = [] - for layout, base in ((_TARGET_LAYOUT, 0), (_DRAFT_LAYOUT, 1000)): - width = layout["max_blocks_per_seq"] - table = torch.arange(num_seqs * width, dtype=torch.int32, device="cuda") - table = table.reshape(num_seqs, width) + base + 100 * generation - table[:, -1] = -1 - # V2 offsets: the key role of the layer's pool holds slot * scale, unallocated - # entries stay -1, and every other pool and role is noise. - offsets = torch.full( - (layout["num_pools"], num_seqs, 2, width), 7777, dtype=torch.int32, device="cuda" - ) - offsets[layout["pool_id"], :, 0, :] = torch.where(table < 0, table, table * layout["scale"]) - slots.append(table) - if layout is _TARGET_LAYOUT: - metadata.kv_cache_block_offsets = offsets - else: - metadata.draft_kv_cache_block_offsets = offsets - return slots - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_qsa_draft_block_table_exists_only_for_a_separate_draft_manager(monkeypatch) -> None: - metadata = _qsa_metadata_with_draft(monkeypatch) - - # Each table is sized, and its pool and page scale taken, from its own manager. - assert metadata.qsa_block_table.shape == (4, _TARGET_LAYOUT["max_blocks_per_seq"]) - assert metadata.qsa_draft_block_table.shape == (4, _DRAFT_LAYOUT["max_blocks_per_seq"]) - assert metadata.qsa_draft_block_table.data_ptr() != metadata.qsa_block_table.data_ptr() - assert (metadata._qsa_attention_pool_id, metadata._qsa_page_index_scale) == (1, 6) - assert (metadata._qsa_draft_attention_pool_id, metadata._qsa_draft_page_index_scale) == (0, 4) +@pytest.mark.parametrize( + "layout, slots", + ((_TARGET_LAYOUT, 2), (dict(_DRAFT_LAYOUT, sparse_layer_ids=[48, 49, 50]), 3)), + ids=("target", "draft"), +) +def test_qsa_shared_topk_slots_follow_the_bound_manager(layout, slots) -> None: + """The shared selection buffers are addressed by the metadata's own manager.""" + metadata = _qsa_metadata_over(layout, mtp_index_share=True) - target_only = _qsa_metadata_with_draft(monkeypatch, draft_layout=None) - assert target_only.qsa_draft_block_table is None - assert target_only.prepare_for_draft_forward() is None + assert metadata.qsa_shared_topk_indices.shape[0] == slots + assert metadata.qsa_shared_topk_visible_blocks.shape[0] == slots -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -@pytest.mark.parametrize("kind", ("not_qsa", "no_local_layer")) -def test_qsa_draft_table_needs_a_qsa_draft_manager_with_local_layers(monkeypatch, kind) -> None: - if kind == "not_qsa": - draft_manager = SimpleNamespace(qsa_position_layer_id=0) - else: - draft_manager = _qsa_layout_manager(**_DRAFT_LAYOUT) - draft_manager.qsa_position_layer_id = None - metadata = _qsa_metadata_with_draft(monkeypatch, draft_manager=draft_manager) - metadata._seq_lens = torch.ones(2, dtype=torch.int32) - - assert metadata.qsa_draft_block_table is None - # The target's own manager is bound, as outside a draft loop, and the hook and the - # refresh both leave the target bindings alone. - metadata.prepare_for_draft_forward() - metadata._refresh_qsa_draft_block_table() - assert metadata.draft_replay_swapped_attrs == {} - assert (metadata._qsa_attention_pool_id, metadata._qsa_page_index_scale) == (1, 6) +def _set_qsa_offsets(metadata, layout: dict, num_seqs: int, generation: int) -> torch.Tensor: + """Write fresh attention offsets for ``metadata``'s manager; return the raw slots its + table must decode to.""" + width = layout["max_blocks_per_seq"] + table = torch.arange(num_seqs * width, dtype=torch.int32, device="cuda") + table = table.reshape(num_seqs, width) + 100 * generation + table[:, -1] = -1 + # V2 offsets: the key role of the layer's pool holds slot * scale, unallocated + # entries stay -1, and every other pool and role is noise. + offsets = torch.full( + (layout["num_pools"], num_seqs, 2, width), 7777, dtype=torch.int32, device="cuda" + ) + offsets[layout["pool_id"], :, 0, :] = torch.where(table < 0, table, table * layout["scale"]) + metadata.kv_cache_block_offsets = offsets + return table -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -@pytest.mark.parametrize( - "draft_layout, slots", - ( - (dict(_DRAFT_LAYOUT, sparse_layer_ids=[48, 49, 50]), 3), # draft wider than target - (_DRAFT_LAYOUT, 2), # target wider than draft - (None, 2), # no separate draft manager - ), - ids=("draft_wider", "target_wider", "no_draft"), +_LAYOUT_CASES = pytest.mark.parametrize( + "layout, pool_id, scale", + ((_TARGET_LAYOUT, 1, 6), (_DRAFT_LAYOUT, 0, 4)), + ids=("target", "draft"), ) -def test_qsa_shared_topk_slots_cover_the_wider_manager(monkeypatch, draft_layout, slots) -> None: - """The shared selection buffers are addressed by whichever manager is bound.""" - metadata = _qsa_metadata_with_draft( - monkeypatch, mtp_index_share=True, draft_layout=draft_layout - ) - - assert metadata.qsa_shared_topk_indices.shape[0] == slots - assert metadata.qsa_shared_topk_visible_blocks.shape[0] == slots @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_qsa_draft_forward_addresses_the_draft_manager_pages(monkeypatch) -> None: - """prepare() decodes both tables; the draft swap makes `qsa_block_table` the draft one - and the restore puts the target's table, pool and scale back.""" - num_seqs = 3 - metadata = _qsa_metadata_with_draft(monkeypatch) - metadata._seq_lens = torch.ones(num_seqs, dtype=torch.int32) - target_slots, draft_slots = _set_qsa_offsets(metadata, num_seqs, generation=0) - target_table = metadata.qsa_block_table - - metadata._refresh_qsa_block_table() - metadata._refresh_qsa_draft_block_table() - torch.testing.assert_close(target_table[:num_seqs], target_slots) - torch.testing.assert_close(metadata.qsa_draft_block_table[:num_seqs], draft_slots) - - draft_manager = metadata.draft_kv_cache_manager - saved = prepare_attn_metadata_for_draft_replay(metadata, draft_manager) - assert saved is not None - assert metadata.kv_cache_manager is draft_manager - assert metadata.qsa_block_table is metadata.qsa_draft_block_table - # Every operand of the table refresh is the draft manager's while swapped, so a - # refresh rebuilds the draft table and leaves the target's alone. - metadata._refresh_qsa_block_table() - torch.testing.assert_close(metadata.qsa_block_table[:num_seqs], draft_slots) - torch.testing.assert_close(target_table[:num_seqs], target_slots) - - restore_attn_metadata_after_draft_replay(metadata, saved) - assert metadata.kv_cache_manager is not draft_manager - assert metadata.qsa_block_table is target_table - assert (metadata._qsa_attention_pool_id, metadata._qsa_page_index_scale) == (1, 6) +@_LAYOUT_CASES +def test_qsa_block_table_follows_the_bound_manager_layout(layout, pool_id, scale) -> None: + """A target metadata and a draft metadata over a separate draft manager each size + their table, and take its pool and page scale, from their own manager.""" + metadata = _qsa_metadata_over(layout) + + assert metadata.qsa_block_table.shape == (4, layout["max_blocks_per_seq"]) + assert (metadata._attention_pool_id, metadata._page_index_scale) == (pool_id, scale) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_qsa_prepare_decodes_the_target_and_the_draft_table(monkeypatch) -> None: - """`prepare()` is what refreshes both tables before every step and replay: a second - generation of offsets reaches the target table and the draft table through it.""" +@_LAYOUT_CASES +def test_qsa_prepare_decodes_the_bound_manager_pages(monkeypatch, layout, pool_id, scale) -> None: + """`prepare()` refreshes the table before every step and replay: each generation of + offsets reaches it, decoded with the bound manager's pool and scale.""" num_seqs = 3 - metadata = _qsa_metadata_with_draft(monkeypatch) + metadata = _qsa_metadata_over(layout) metadata._seq_lens = torch.ones(num_seqs, dtype=torch.int32) - # The parent's prepare and the row mapping are not under test here. - monkeypatch.setattr(TrtllmAttentionMetadata, "prepare", lambda self: None) - monkeypatch.setattr(QSAAttentionMetadata, "_refresh_qsa_token_mapping", lambda self: None) - monkeypatch.setattr( - QSAAttentionMetadata, "_refresh_qsa_speculative_snapshot_state", lambda self: None - ) + metadata._num_contexts = 0 + qsa = metadata.find_ext(QSAMetadata) + # The row mapping is not under test here. + monkeypatch.setattr(qsa, "_refresh_token_mapping", lambda: None) for generation in (0, 1): - target_slots, draft_slots = _set_qsa_offsets(metadata, num_seqs, generation) - metadata.prepare() - torch.testing.assert_close(metadata.qsa_block_table[:num_seqs], target_slots) - torch.testing.assert_close(metadata.qsa_draft_block_table[:num_seqs], draft_slots) + slots = _set_qsa_offsets(metadata, layout, num_seqs, generation) + qsa.prepare() + torch.testing.assert_close(metadata.qsa_block_table[:num_seqs], slots) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_qsa_draft_forward_refuses_an_unswapped_manager(monkeypatch) -> None: - metadata = _qsa_metadata_with_draft(monkeypatch) - - with pytest.raises(RuntimeError, match="draft KV cache manager"): - metadata.prepare_for_draft_forward() - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_qsa_draft_table_replays_the_refreshed_draft_pages(monkeypatch) -> None: - """A graph captured across the draft swap reads each table by address: the trunk - segment the target table, the draft segment the draft one, and a refresh outside - capture (as prepare() does before every replay) is what the replay sees.""" +def test_qsa_block_tables_replay_the_refreshed_pages() -> None: + """A graph that reads the target table and then a draft metadata's table reads each + by address, so a refresh outside capture (as prepare() does before every replay) is + what the replay sees, and the two tables never alias.""" num_seqs = 3 - metadata = _qsa_metadata_with_draft(monkeypatch) - metadata._seq_lens = torch.ones(num_seqs, dtype=torch.int32) - _set_qsa_offsets(metadata, num_seqs, generation=0) - metadata._refresh_qsa_block_table() - metadata._refresh_qsa_draft_block_table() - draft_manager = metadata.draft_kv_cache_manager - target_seen = torch.zeros_like(metadata.qsa_block_table[:num_seqs]) - draft_seen = torch.zeros_like(metadata.qsa_draft_block_table[:num_seqs]) + target, draft = _qsa_metadata_over(_TARGET_LAYOUT), _qsa_metadata_over(_DRAFT_LAYOUT) + tables = [] + for metadata in (target, draft): + metadata._seq_lens = torch.ones(num_seqs, dtype=torch.int32) + tables.append(metadata.qsa_block_table) + assert tables[0].data_ptr() != tables[1].data_ptr() + seen = [torch.zeros_like(table[:num_seqs]) for table in tables] graph = torch.cuda.CUDAGraph() torch.cuda.synchronize() with torch.cuda.graph(graph): - target_seen.copy_(metadata.qsa_block_table[:num_seqs]) - saved = prepare_attn_metadata_for_draft_replay(metadata, draft_manager) - draft_seen.copy_(metadata.qsa_block_table[:num_seqs]) - restore_attn_metadata_after_draft_replay(metadata, saved) - - target_slots, draft_slots = _set_qsa_offsets(metadata, num_seqs, generation=1) - metadata._refresh_qsa_block_table() - metadata._refresh_qsa_draft_block_table() + for table, out in zip(tables, seen): + out.copy_(table[:num_seqs]) + + expected = [] + for metadata, layout in ((target, _TARGET_LAYOUT), (draft, _DRAFT_LAYOUT)): + expected.append(_set_qsa_offsets(metadata, layout, num_seqs, generation=1)) + metadata._refresh_block_table() graph.replay() torch.cuda.synchronize() - torch.testing.assert_close(target_seen, target_slots) - torch.testing.assert_close(draft_seen, draft_slots) + for out, slots in zip(seen, expected): + torch.testing.assert_close(out, slots) diff --git a/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py b/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py index 4b93cd54dcea..8480ebd9cade 100644 --- a/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py +++ b/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py @@ -5,6 +5,7 @@ import pytest import torch from utils.llm_data import llm_models_root +from utils.metadata_contract import check_copy_reproduces_creation_config from utils.util import getSMVersion import tensorrt_llm @@ -12,6 +13,7 @@ from tensorrt_llm._torch.attention.backends.interface import AttentionForwardArgs from tensorrt_llm._torch.attention.backends.sparse.rocket import ( RocketKVCacheManager, + RocketMetadata, RocketTrtllmAttention, RocketTrtllmAttentionMetadata, RocketVanillaAttention, @@ -557,7 +559,8 @@ def test_sparse_attn_predict(batch_size, num_contexts): for req_idx in range(num_contexts, batch_size): # Get the number of KT tokens for this request - past_token = past_seen_tokens[req_idx] + # The cached length the kernels see, capped at the prompt budget. + past_token = trtllm_metadata.kv_cache_params.num_cached_tokens_per_seq[req_idx] num_kt_tokens = ( past_token + 1 + sparse_attn_config_trtllm.page_size - 1 ) // sparse_attn_config_trtllm.page_size @@ -614,7 +617,7 @@ def test_sparse_attn_predict(batch_size, num_contexts): q = q.view(1, seq_len, num_heads, head_dim).transpose(1, 2) k = k.view(1, seq_len, num_kv_heads, head_dim) - past_seen_token = past_seen_tokens[i] + past_seen_token = vanilla_metadata.kv_cache_params.num_cached_tokens_per_seq[i] vanilla_indices = vanilla_attn._rocketkv_selection( q, k, vanilla_metadata, past_seen_token, i ) @@ -771,6 +774,74 @@ def test_rocket_add_dummy_requests_forwards_capture_sampling_params(mocker): assert base_add_dummy_requests.call_args.kwargs["capture_sampling_params"] is sampling_params +@pytest.mark.parametrize( + "metadata_cls", [RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata] +) +def test_rocket_copies_reproduce_creation_config(metadata_cls): + """A constructed copy of a prepared Rocket metadata reports its creation configuration.""" + sparse_attn_config = RocketSparseAttentionConfig( + window_size=32, kernel_size=3, prompt_budget=256, page_size=4, kt_cache_dtype="bfloat16" + ) + seq_lens, past_seen_tokens, request_ids = [300, 1], [0, 150], [0, 1] + kv_cache_manager = create_rocket_kv_cache_manager( + 1, 8, 128, 64, 4096, 2, DataType.BF16, sparse_attn_config + ) + kv_cache_manager.add_dummy_requests( + request_ids, [seq + past for seq, past in zip(seq_lens, past_seen_tokens)] + ) + try: + metadata = create_test_metadata( + seq_lens, + 1, + past_seen_tokens, + request_ids, + kv_cache_manager, + sparse_attn_config, + metadata_cls, + ) + assert metadata.find_ext(RocketMetadata) is not None + check_copy_reproduces_creation_config(metadata) + finally: + kv_cache_manager.shutdown() + + +@pytest.mark.parametrize( + "metadata_cls", [RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata] +) +def test_rocket_prompt_budget_holds_across_repeated_prepares(metadata_cls): + """Preparing the same batch again (as CUDA graph warmup does) caps the cached + lengths at the prompt budget once, not once per prepare.""" + sparse_attn_config = RocketSparseAttentionConfig( + window_size=32, kernel_size=3, prompt_budget=256, page_size=4, kt_cache_dtype="bfloat16" + ) + seq_lens, past_seen_tokens, request_ids = [300, 1], [0, 400], [0, 1] + kv_cache_manager = create_rocket_kv_cache_manager( + 1, 8, 128, 64, 4096, 2, DataType.BF16, sparse_attn_config + ) + kv_cache_manager.add_dummy_requests( + request_ids, [seq + past for seq, past in zip(seq_lens, past_seen_tokens)] + ) + try: + metadata = create_test_metadata( + seq_lens, + 1, + past_seen_tokens, + request_ids, + kv_cache_manager, + sparse_attn_config, + metadata_cls, + ) + # The generation request's 401-token prompt is cut to the 256-token budget. + expected_cached = [0, 400 + 256 - 401] + for _ in range(3): + assert list(metadata.kv_cache_params.num_cached_tokens_per_seq) == expected_cached + if metadata_cls is RocketTrtllmAttentionMetadata: + assert metadata.kv_lens[:2].tolist() == [300, expected_cached[1] + 1] + metadata.prepare() + finally: + kv_cache_manager.shutdown() + + if __name__ == "__main__": # RocketKV e2e tests print("=== Testing RocketKV E2E tests ===") diff --git a/tests/unittest/_torch/attention/sparse/test_sparse_attention.py b/tests/unittest/_torch/attention/sparse/test_sparse_attention.py index 0d8aa338747b..31132a710379 100644 --- a/tests/unittest/_torch/attention/sparse/test_sparse_attention.py +++ b/tests/unittest/_torch/attention/sparse/test_sparse_attention.py @@ -25,8 +25,10 @@ import pytest import torch +from _torch.helpers import bare_extension from tensorrt_llm._torch.attention.backends import trtllm as trtllm_backend +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionInputType, @@ -312,12 +314,11 @@ def _make_sparse_prediction_forward_metadata() -> TrtllmAttentionMetadata: metadata._seq_lens_kv = seq_lens metadata._seq_lens_cuda = None metadata.kv_cache_manager = None - metadata._max_seq_len_storage = 4 + metadata.core()._max_seq_len_storage = 4 metadata.use_paged_context_fmha = False metadata.cu_q_seqlens = None metadata.cu_kv_seqlens = None - metadata.enable_flash_mla = False - metadata.spec_bl_tree_first_sparse_mask_offset_kv = None + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata.kv_lens_cuda_runtime = torch.tensor([2], dtype=torch.int32) metadata.kv_lens_runtime = torch.tensor([2], dtype=torch.int32) metadata.prompt_lens_cuda_runtime = torch.tensor([2], dtype=torch.int32) diff --git a/tests/unittest/_torch/attention/sparse/test_sparse_mha.py b/tests/unittest/_torch/attention/sparse/test_sparse_mha.py index fea130f5656c..1a053f96d70d 100644 --- a/tests/unittest/_torch/attention/sparse/test_sparse_mha.py +++ b/tests/unittest/_torch/attention/sparse/test_sparse_mha.py @@ -33,17 +33,16 @@ from utils.util import getSMVersion import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features.spec_decoding import ( + generate_spec_decoding_packed_mask, + generate_spec_decoding_position_offsets, +) from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionRuntimeFeatures, ) from tensorrt_llm._torch.attention.backends.sparse.params import SparseParams -from tensorrt_llm._torch.attention.backends.trtllm import ( - TrtllmAttention, - TrtllmAttentionMetadata, - generate_spec_decoding_packed_mask, - generate_spec_decoding_position_offsets, -) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._utils import str_dtype_to_binding, torch_dtype_to_str @@ -275,12 +274,12 @@ def create_generation_inputs(scenario: MhaGenerationScenario) -> MhaGenerationIn runtime_features=AttentionRuntimeFeatures( has_speculative_draft_tokens=scenario.has_draft_tokens ), - is_spec_decoding_enabled=scenario.has_draft_tokens, - use_spec_decoding=scenario.has_draft_tokens, - is_spec_dec_tree=False, - max_total_draft_tokens=( - scenario.max_query_len - 1 if scenario.has_draft_tokens else None - ), + ) + metadata.is_spec_decoding_enabled = scenario.has_draft_tokens + metadata.use_spec_decoding = scenario.has_draft_tokens + metadata.is_spec_dec_tree = False + metadata.max_total_draft_tokens = ( + scenario.max_query_len - 1 if scenario.has_draft_tokens else None ) if scenario.has_draft_tokens: draft_len = scenario.max_query_len - 1 diff --git a/tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py b/tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py index b020bf43c7bc..7777d7f6cad8 100644 --- a/tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py +++ b/tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py @@ -1355,7 +1355,7 @@ def populate_gen_dsa_kv_cache(mla: MLA, AttentionCls: AttentionBackend, num_cached_tokens_per_seq=[0] * len(gen_with_cache)), mapping=mapping, - sparse_attention_config=sparse_config, + sparse_metadata_params=sparse_config.to_sparse_metadata_params(), ) cached_metadata.prepare() @@ -1554,7 +1554,7 @@ def populate_gen_deepseek_v4_kv_cache( len(gen_with_cache) # No prior cache ), mapping=mapping, - sparse_attention_config=sparse_config, + sparse_metadata_params=sparse_config.to_sparse_metadata_params(), ) cached_metadata.prepare() _ = mla_forward_impl_with_deepseek_v4_wo_linear( diff --git a/tests/unittest/_torch/attention/sparse/test_sparse_mqa_gqa.py b/tests/unittest/_torch/attention/sparse/test_sparse_mqa_gqa.py index 5ae05df42a90..be5290fe5cf8 100644 --- a/tests/unittest/_torch/attention/sparse/test_sparse_mqa_gqa.py +++ b/tests/unittest/_torch/attention/sparse/test_sparse_mqa_gqa.py @@ -39,6 +39,10 @@ from utils.util import getSMVersion import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features.spec_decoding import ( + generate_spec_decoding_packed_mask, + generate_spec_decoding_position_offsets, +) from tensorrt_llm._torch.attention.backends.fmha.msa_prefill import run_msa_sparse_gqa from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, @@ -51,12 +55,7 @@ msa_package_available, ) from tensorrt_llm._torch.attention.backends.sparse.params import SparseParams -from tensorrt_llm._torch.attention.backends.trtllm import ( - TrtllmAttention, - TrtllmAttentionMetadata, - generate_spec_decoding_packed_mask, - generate_spec_decoding_position_offsets, -) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._utils import str_dtype_to_binding, torch_dtype_to_str @@ -283,11 +282,8 @@ def indices_block_size(self) -> int: return 1 -@dataclass class _SparseMqaGqaMetadata(TrtllmAttentionMetadata): - """Attention metadata extended with the static sparse Top-K.""" - - num_sparse_topk: int = 64 + """Attention metadata of the static sparse Top-K, given as ``num_sparse_topk``.""" class _StaticSparseMqaGqaAttention(TrtllmAttention): @@ -1599,11 +1595,11 @@ def _create_generation_inputs(s: GenerationScenario) -> _GenerationInputs: num_sparse_topk=num_sparse_topk, max_num_heads_per_kv=s.q_heads_per_kv_head, runtime_features=AttentionRuntimeFeatures(has_speculative_draft_tokens=s.has_draft_tokens), - is_spec_decoding_enabled=s.has_draft_tokens, - use_spec_decoding=s.has_draft_tokens, - is_spec_dec_tree=False, - max_total_draft_tokens=s.max_query_len - 1 if s.has_draft_tokens else None, ) + metadata.is_spec_decoding_enabled = s.has_draft_tokens + metadata.use_spec_decoding = s.has_draft_tokens + metadata.is_spec_dec_tree = False + metadata.max_total_draft_tokens = s.max_query_len - 1 if s.has_draft_tokens else None if s.has_draft_tokens: draft_len = s.max_query_len - 1 metadata.spec_decoding_position_offsets = generate_spec_decoding_position_offsets( diff --git a/tests/unittest/_torch/attention/test_attention_failure_logging.py b/tests/unittest/_torch/attention/test_attention_failure_logging.py index 97b0f31bac49..386889bd25d6 100644 --- a/tests/unittest/_torch/attention/test_attention_failure_logging.py +++ b/tests/unittest/_torch/attention/test_attention_failure_logging.py @@ -25,8 +25,10 @@ import pytest import torch +from _torch.helpers import bare_extension from tensorrt_llm._torch.attention.backends import utils +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, PredefinedAttentionMask, @@ -204,9 +206,7 @@ def forward(self, *args, **kwargs): metadata.workspace = torch.zeros(1024, dtype=torch.int8) metadata.cu_q_seqlens = torch.zeros(batch + 1, dtype=torch.int32) metadata.cu_kv_seqlens = torch.zeros(batch + 1, dtype=torch.int32) - metadata.enable_flash_mla = False - metadata.spec_bl_tree_first_sparse_mask_offset_kv = None - metadata.spec_decoding_bl_tree_mask = None + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata.max_context_q_len_override = None metadata.kv_cache_manager = SimpleNamespace( max_attention_window_vec=[2048, None], diff --git a/tests/unittest/_torch/attention/test_attention_mla.py b/tests/unittest/_torch/attention/test_attention_mla.py index 10c4e95c6a8f..3d2d4cb00c10 100644 --- a/tests/unittest/_torch/attention/test_attention_mla.py +++ b/tests/unittest/_torch/attention/test_attention_mla.py @@ -9,13 +9,15 @@ import pytest import torch +from _torch.helpers import refresh_from_lengths import tensorrt_llm from tensorrt_llm._torch.attention import mla as mla_module +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata from tensorrt_llm._torch.attention.backends.interface import ( AttentionInputType, MLAParams, PositionalEmbeddingParams, RopeParams) -from tensorrt_llm._torch.attention.backends.trtllm import \ - TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import ( + TrtllmAttentionMetadata, TrtllmCoreMetadata) from tensorrt_llm._torch.attention.backends.utils import get_attention_backend from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import \ @@ -1196,9 +1198,9 @@ def yarn_get_mscale(scale=1, mscale=1): ), mapping=mapping, ) - if backend_name == "TRTLLM": - gen_metadata_kwargs["enable_flash_mla"] = ( - cuda_capability == (9, 0)) + if backend_name == "TRTLLM" and cuda_capability == (9, 0): + gen_metadata_kwargs["extension_types"] = ( + FlashMlaMetadata, ) attn_metadata = AttentionCls.Metadata(**gen_metadata_kwargs) attn_metadata.prepare() for layer_idx in range(num_layers): @@ -1440,21 +1442,24 @@ def yarn_get_mscale(scale=1, mscale=1): NUM_HEADS = 128 -class _Prefixes: - """The state `mla_prepare_*` touches, with the real methods bound to it. +class _PrefixesCore(TrtllmCoreMetadata): + """The core state `mla_prepare_*` reads, with the length properties as plain attributes.""" + + num_contexts = 0 + num_seqs = 0 + seq_lens_cuda = None + + +class _Prefixes(TrtllmAttentionMetadata): + """The state `mla_prepare_*` touches, running the real metadata methods. Deliberately not a full `TrtllmAttentionMetadata`: that needs a KV cache - manager and a resource pool, none of which these methods read. Binding the - production functions keeps this a test of the shipped code rather than of a - reimplementation. + manager and a resource pool, none of which these methods read. Inheriting + the production methods keeps this a test of the shipped code rather than of + a reimplementation. """ - # Bind the unbound functions so any edit to them is exercised here. - mla_prepare_scheduler_buffers = TrtllmAttentionMetadata.mla_prepare_scheduler_buffers - mla_prepare_ctx_cu_seqlens = TrtllmAttentionMetadata.mla_prepare_ctx_cu_seqlens - _invalidate_mla_scheduler_buffers = TrtllmAttentionMetadata._invalidate_mla_scheduler_buffers - on_update_kv_lens = TrtllmAttentionMetadata.on_update_kv_lens - update_for_spec_dec = TrtllmAttentionMetadata.update_for_spec_dec + core_metadata_type = _PrefixesCore def __init__(self, q_lens, kv_lens, num_contexts=0): device = torch.device("cuda") @@ -1467,17 +1472,16 @@ def __init__(self, q_lens, kv_lens, num_contexts=0): dtype=torch.int32, device=device) size = self.num_seqs + 1 - self.mla_cu_q_rows = torch.zeros(size, dtype=torch.int32, device=device) - self.mla_cu_kv_seqlens = torch.zeros(size, - dtype=torch.int32, - device=device) - self.mla_ctx_cu_q_seqlens = torch.zeros(size, + self.core().mla_cu_q_rows = torch.zeros(size, dtype=torch.int32, device=device) - # The hooks also read `enable_flash_mla` and `mixed_precision_staging`. - self.enable_flash_mla = False - self.mixed_precision_staging = None - self._invalidate_mla_scheduler_buffers() + self.core().mla_cu_kv_seqlens = torch.zeros(size, + dtype=torch.int32, + device=device) + self.core().mla_ctx_cu_q_seqlens = torch.zeros(size, + dtype=torch.int32, + device=device) + self.core()._invalidate_mla_scheduler_buffers() def rebuild(self): cu_q, cu_kv = self.mla_prepare_scheduler_buffers(NUM_HEADS) @@ -1495,18 +1499,18 @@ def test_q_rows_follow_a_spec_dec_query_length_change(): cu_q, _ = state.rebuild() assert cu_q == [0, 4 * NUM_HEADS, 8 * NUM_HEADS] - # What MTPWorker.change_attn_metadata does to the generation rows. + # What MTPWorker.change_attn_metadata does to the generation rows of its draft. state.seq_lens_cuda -= 1 assert state.rebuild( )[0] == cu_q, "no rebuild is expected until a hook fires" - state.update_for_spec_dec() + refresh_from_lengths(state) assert state.rebuild()[0] == [0, 3 * NUM_HEADS, 6 * NUM_HEADS] @pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device") def test_kv_prefix_follows_an_in_place_device_kv_len_change(): - """The overlap scheduler bumps `kv_lens_cuda` on device, never the host copy. + """A draft step advances `kv_lens_cuda` on device, never the host copy. A host-derived prefix cannot see this, which is why the rebuild reads `kv_lens_cuda` directly. @@ -1514,8 +1518,10 @@ def test_kv_prefix_follows_an_in_place_device_kv_len_change(): state = _Prefixes(q_lens=[1, 1], kv_lens=[100, 200]) assert state.rebuild()[1] == [0, 100, 300] - state.kv_lens_cuda += torch.tensor([3, 5], dtype=torch.int32, device="cuda") - state.on_update_kv_lens() + state.core().kv_lens_cuda += torch.tensor([3, 5], + dtype=torch.int32, + device="cuda") + refresh_from_lengths(state) assert state.rebuild()[1] == [0, 103, 308] @@ -1527,7 +1533,7 @@ def test_partial_acceptance_differs_per_request(): state.seq_lens_cuda.copy_( torch.tensor([1, 3, 4], dtype=torch.int32, device="cuda")) - state.update_for_spec_dec() + refresh_from_lengths(state) cu_q, _ = state.rebuild() assert cu_q == [0, 1 * NUM_HEADS, 4 * NUM_HEADS, 8 * NUM_HEADS] @@ -1540,7 +1546,7 @@ def test_context_prefix_counts_tokens_not_rows(): state.seq_lens_cuda.copy_( torch.tensor([2, 3, 7], dtype=torch.int32, device="cuda")) - state.on_update_kv_lens() + refresh_from_lengths(state) assert state.mla_prepare_ctx_cu_seqlens().tolist() == [0, 2, 5, 12] diff --git a/tests/unittest/_torch/attention/test_cute_dsl_mla.py b/tests/unittest/_torch/attention/test_cute_dsl_mla.py index be9bd11ac0c5..5b3bbd7e4814 100644 --- a/tests/unittest/_torch/attention/test_cute_dsl_mla.py +++ b/tests/unittest/_torch/attention/test_cute_dsl_mla.py @@ -3,15 +3,18 @@ """Host-side contract tests for the CuTe DSL MLA FMHA library (FP8 scales, split-KV sizing).""" from types import SimpleNamespace +from unittest.mock import Mock import pytest import torch +from _torch.helpers import bare_extension from tensorrt_llm._torch.attention.backends.fmha.cute_dsl_mla import CuteDslMlaFmha from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionInputType, ) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmCoreMetadata from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from tensorrt_llm.bindings import DataType @@ -490,7 +493,10 @@ def test_generation_kv_lengths_keep_the_runner_alignment(num_contexts: int) -> N @pytest.mark.parametrize("num_contexts", [0, 1, 3, 4]) def test_decode_kv_lengths_keep_one_pointer_per_step(num_contexts: int) -> None: """Every layer of a step passes the same pointer, so the split plan runs once.""" - meta = SimpleNamespace(max_num_requests=16) + core = bare_extension( + TrtllmCoreMetadata, Mock(), max_num_requests=16, _aligned_kv_lens_staging=None + ) + meta = SimpleNamespace(aligned_kv_lens_staging=core.aligned_kv_lens_staging) sequence_lengths = torch.arange(1, 17, dtype=torch.int32) generation_lengths = sequence_lengths[num_contexts:] @@ -502,7 +508,7 @@ def test_decode_kv_lengths_keep_one_pointer_per_step(num_contexts: int) -> None: torch.testing.assert_close(second_layer, generation_lengths) if generation_lengths.data_ptr() % 16 == 0: assert first_layer is generation_lengths - assert not hasattr(meta, "_cute_dsl_mla_kv_lens_staging") + assert core._aligned_kv_lens_staging is None @pytest.mark.parametrize( diff --git a/tests/unittest/_torch/attention/test_cute_dsl_mla_split_plan.py b/tests/unittest/_torch/attention/test_cute_dsl_mla_split_plan.py index 566fe85dd9e4..22eed8039828 100644 --- a/tests/unittest/_torch/attention/test_cute_dsl_mla_split_plan.py +++ b/tests/unittest/_torch/attention/test_cute_dsl_mla_split_plan.py @@ -2,12 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 """Per-request split-KV plan of the CuTe DSL MLA decode: policy, kernel vs mirror, kernel output.""" +import functools import math from types import SimpleNamespace import pytest import torch +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmCoreMetadata from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from tensorrt_llm._utils import get_sm_version @@ -88,7 +90,13 @@ def test_split_kv_plan_is_launched_once_per_step(monkeypatch) -> None: monkeypatch.setattr( mod, "launch_mla_split_kv_plan", lambda *args, **kwargs: launches.append(kwargs) ) - meta = SimpleNamespace(max_num_requests=16, _cute_dsl_mla_split_plan_key=None) + meta = SimpleNamespace( + max_num_requests=16, _cute_dsl_mla_split_plan=None, _cute_dsl_mla_split_plan_key=None + ) + # The plan cache is the core's: run its method over the stub's fields. + meta.cute_dsl_mla_split_plan = functools.partial( + TrtllmCoreMetadata.cute_dsl_mla_split_plan, meta + ) cache_seqs = torch.full((8,), 14208, dtype=torch.int32, device="cuda") first = mod.CuteDslMlaFmha._split_kv_plan(meta, cache_seqs, 8, 8, 96) assert first.numel() == 8 and meta._cute_dsl_mla_split_plan.numel() == 16 diff --git a/tests/unittest/_torch/attention/test_draft_metadata_reuse.py b/tests/unittest/_torch/attention/test_draft_metadata_reuse.py new file mode 100644 index 000000000000..12bc4a5041d8 --- /dev/null +++ b/tests/unittest/_torch/attention/test_draft_metadata_reuse.py @@ -0,0 +1,780 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A one-model draft metadata takes its target's prepared state. + +The draft (``md.draft``) of a target that shares its KV cache binds the +target's derived state when its loop starts and copies the tensors a draft +loop writes into its own buffers. For the dense TRTLLM backend and the DSA, +DeepSeek-V4 and CSA2 sparse families these tests check that the result equals +a draft that derives its own state in the target's prepare, that a draft step +leaves the target untouched, and that no draft tensor shares storage with a +buffer the target keeps apart from the draft. +""" + +import dataclasses +from types import SimpleNamespace + +import pytest +import torch +from _torch.helpers import refresh_from_lengths, start_draft_loop +from utils.metadata_contract import check_copy_reproduces_creation_config +from utils.util import getSMVersion + +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig +from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures, BatchInputs +from tensorrt_llm._torch.metadata import KVCacheParams +from tensorrt_llm.bindings import DataType +from tensorrt_llm.bindings.internal.batch_manager import CacheType +from tensorrt_llm.mapping import Mapping + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +MAX_DRAFT_TOKENS = 3 +HISTORY = 300 + +# Bookkeeping of the adoption itself, present only on an adopting draft, and +# DSA's log-once marker of the fused decode-metadata kernel, set by whichever +# metadata ran that kernel first. +_ADOPTION_BOOKKEEPING = {"_draft_adopted_names", "_draft_copy_plan", "_fused_dsa_meta_armed"} + + +# --- Comparing metadata state ------------------------------------------------- + + +def _extensions_by_type(metadata): + return {type(extension): extension for extension in metadata._attached_extensions()} + + +def _tensor_items(name, value): + """``(key, tensor)`` of a tensor attribute or of each tensor in a dict attribute.""" + if isinstance(value, torch.Tensor): + yield name, value + elif isinstance(value, dict): + for key, item in value.items(): + if isinstance(item, torch.Tensor): + yield (name, key), item + + +def _extension_tensors(metadata): + """Every tensor of every extension, by (extension name, attribute key).""" + found = {} + for extension_type, extension in _extensions_by_type(metadata).items(): + for name, value in vars(extension).items(): + for key, tensor in _tensor_items(name, value): + found[(extension_type.__name__, key)] = tensor + return found + + +def _zero_buffers(metadata): + """Zero the uninitialized buffers so untouched regions compare equal.""" + for tensor in _extension_tensors(metadata).values(): + if tensor.is_floating_point() or tensor.dtype in ( + torch.int32, + torch.int64, + torch.int8, + torch.uint8, + torch.bool, + ): + tensor.zero_() + + +def _comparable(value): + if isinstance(value, torch.Tensor): + return ("tensor", value.shape, value.dtype, value.detach().cpu().clone()) + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, (list, tuple, set, frozenset)) and all( + item is None or isinstance(item, (bool, int, float, str, tuple)) for item in value + ): + if isinstance(value, (set, frozenset)): + return tuple(sorted(value, key=repr)) + return tuple(value) + return None + + +def comparable_state(metadata): + """The comparable attributes of every extension; dictionaries entry by entry.""" + found = {} + for extension_type, extension in _extensions_by_type(metadata).items(): + for name, value in vars(extension).items(): + entries = ( + {(name, key): item for key, item in value.items()} + if isinstance(value, dict) + else {name: value} + ) + for key, item in entries.items(): + comparable = _comparable(item) + if comparable is not None: + found[(extension_type.__name__, key)] = comparable + return found + + +def _state_differences(actual, expected): + """The keys whose values differ or that only one of the states has.""" + different = [] + for key in sorted(set(actual) | set(expected), key=repr): + if key[1] in _ADOPTION_BOOKKEEPING: + continue + if key not in actual or key not in expected: + different.append(key) + continue + left, right = actual[key], expected[key] + if isinstance(left, tuple) and left and left[0] == "tensor": + if not ( + isinstance(right, tuple) + and right + and right[0] == "tensor" + and left[1:3] == right[1:3] + and torch.equal(left[3], right[3]) + ): + different.append(key) + elif left != right: + different.append(key) + return different + + +# --- Driving a target and its draft -------------------------------------------- + + +def _prepared(family, *, reuse, start_draft=True, batch=None, **creation_changes): + """A prepared one-model target, whose draft loop started with ``start_draft``. + + ``reuse=False`` makes its draft derive its own state in the target's + prepare; a draft taking the target's state takes it when its loop starts. + """ + metadata = family.backend.Metadata(**{**family.creation, **creation_changes}) + if not reuse: + metadata._draft_adopts = False + _zero_buffers(metadata) + _zero_buffers(metadata.draft) + metadata.prepare(batch or family.batch) + if start_draft: + start_draft_loop(metadata) + torch.cuda.synchronize() + return metadata + + +def _prepare_forward(family, metadata): + """What a forward builds before its first attention layer.""" + if family.num_heads is not None: + metadata.core().mla_prepare_scheduler_buffers(family.num_heads) + if family.forward is not None: + family.forward(metadata) + + +def _draft_step(family, draft, target): + """One MTP draft sub-step: one query token per request, one more KV token.""" + rows = slice(0, len(family.batch.request_ids)) + draft.core().copy_lengths_from(target.core()) + draft.core().set_seq_lens(rows, host=1, device=1, update_counts=True) + draft.core().adjust_kv_lens(rows, 1) + refresh_from_lengths(draft) + _prepare_forward(family, draft) + torch.cuda.synchronize() + + +# --- Families ------------------------------------------------------------------- + + +def _family(backend, creation, batch, **settings): + defaults = dict( + num_heads=None, + state=comparable_state, + banks=frozenset(), + forward=None, + extension_names=("TrtllmCoreMetadata",), + draft_manager=None, + ) + return SimpleNamespace( + backend=backend, creation=creation, batch=batch, **{**defaults, **settings} + ) + + +def _creation(manager, batch_size, **extra): + return dict( + max_num_requests=batch_size, + max_num_tokens=batch_size * (1 + MAX_DRAFT_TOKENS), + max_num_sequences=batch_size, + kv_cache_manager=manager, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + runtime_features=AttentionRuntimeFeatures(has_speculative_draft_tokens=True), + spec_decoding=SpecDecodingConfig( + max_draft_len=MAX_DRAFT_TOKENS, max_total_draft_tokens=MAX_DRAFT_TOKENS + ), + one_model_draft=True, + **extra, + ) + + +def _batch(request_ids, cached): + """A generation batch; different cached lengths per request, so a mixed-up row shows.""" + return BatchInputs( + seq_lens=torch.full((len(request_ids),), 1 + MAX_DRAFT_TOKENS, dtype=torch.int), + num_contexts=0, + request_ids=list(request_ids), + prompt_lens=[HISTORY] * len(request_ids), + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached), + ) + + +def _dense_manager(batch_size, leading_request_ids=()): + from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager + from tensorrt_llm.llmapi.llm_args import KvCacheConfig + + tokens_per_block = 32 + manager = KVCacheManager( + KvCacheConfig(max_tokens=(batch_size + 1) * 512, enable_block_reuse=False), + CacheType.SELF, + num_layers=1, + num_kv_heads=2, + head_dim=64, + tokens_per_block=tokens_per_block, + max_seq_len=512, + max_batch_size=batch_size + 1, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + dtype=DataType.BF16, + ) + for ids in (list(leading_request_ids), list(range(batch_size))): + if ids: + manager.add_dummy_requests( + request_ids=ids, + token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * len(ids), + is_gen=False, + ) + return manager + + +@pytest.fixture(scope="module") +def dense(): + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention + + batch_size = 4 + manager = _dense_manager(batch_size) + # A leading request moves the batch's blocks away from the target's. + draft_manager = _dense_manager(batch_size, leading_request_ids=[batch_size]) + yield _family( + TrtllmAttention, + _creation(manager, batch_size), + _batch(range(batch_size), [HISTORY - 7 * i for i in range(batch_size)]), + draft_manager=draft_manager, + ) + manager.shutdown() + draft_manager.shutdown() + + +@pytest.fixture(scope="module") +def dsa(): + from tensorrt_llm._torch.attention.backends.sparse.dsa import DSACacheManager + from tensorrt_llm._torch.attention.backends.utils import get_attention_backend + from tensorrt_llm._torch.model_config import ModelConfig + from tensorrt_llm.llmapi.llm_args import DeepSeekSparseAttentionConfig, KvCacheConfig + from tensorrt_llm.models.modeling_utils import QuantConfig + + batch_size, tokens_per_block = 8, 64 + mapping = Mapping(world_size=1, tp_size=1, rank=0) + sparse_config = DeepSeekSparseAttentionConfig( + index_n_heads=64, index_head_dim=128, index_topk=2048 + ) + pretrained_config = SimpleNamespace(rms_norm_eps=1e-6) + model_config = ModelConfig( + mapping=mapping, + sparse_attention_config=sparse_config, + pretrained_config=pretrained_config, + quant_config=QuantConfig(), + ) + model_config.extra_attrs["kv_cache_dtype"] = "auto" + manager = DSACacheManager( + KvCacheConfig( + max_tokens=batch_size * (HISTORY + 2 * tokens_per_block), + enable_block_reuse=False, + event_buffer_max_size=0, + ), + CacheType.SELFKONLY, + num_layers=2, + num_kv_heads=1, + head_dim=576, + tokens_per_block=tokens_per_block, + max_seq_len=1024, + max_batch_size=batch_size, + mapping=mapping, + dtype=DataType.BF16, + sparse_attention_config=sparse_config, + model_config=model_config, + vocab_size=129280, + ) + manager.add_dummy_requests( + request_ids=list(range(batch_size)), + token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * batch_size, + is_gen=False, + ) + backend = get_attention_backend( + "TRTLLM", sparse_params=sparse_config.to_sparse_params(pretrained_config=pretrained_config) + ) + creation = _creation( + manager, + batch_size, + enable_context_mla_with_cached_kv=False, + sparse_metadata_params=sparse_config.to_sparse_metadata_params( + pretrained_config=pretrained_config + ), + max_num_heads_per_kv=128, + ) + yield _family( + backend, + creation, + _batch(range(batch_size), [HISTORY - 7 * i for i in range(batch_size)]), + num_heads=128, + ) + manager.shutdown() + + +def _deepseek_v4_manager(batch_size, leading_request_ids=()): + from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import DeepseekV4CacheManager + from tensorrt_llm.llmapi.llm_args import KvCacheConfig, MTPDecodingConfig + + max_seq_len = 1024 + # The cache manager extends its config's compress ratios by the MTP layers. + sparse_config = _deepseek_v4_sparse_config() + manager = DeepseekV4CacheManager( + kv_cache_config=KvCacheConfig( + max_tokens=(batch_size + 1) * max_seq_len, + enable_block_reuse=False, + event_buffer_max_size=0, + ), + kv_cache_type=CacheType.SELFKONLY, + num_layers=len(sparse_config.compress_ratios), + num_kv_heads=1, + head_dim=512, + tokens_per_block=128, + max_seq_len=max_seq_len, + # Room for a draft cache's leading request. + max_batch_size=batch_size + 1, + max_input_len=max_seq_len, + mapping=Mapping(world_size=1, rank=0, tp_size=1, pp_size=1), + dtype=DataType.BF16, + compressor_dtype=DataType.FLOAT, + vocab_size=129280, + max_num_tokens=batch_size * max_seq_len, + sparse_attn_config=sparse_config, + spec_config=MTPDecodingConfig(max_draft_len=MAX_DRAFT_TOKENS), + ) + for ids in (list(leading_request_ids), list(range(batch_size))): + if ids: + manager.add_dummy_requests( + request_ids=ids, + token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * len(ids), + is_gen=False, + ) + return manager + + +def _deepseek_v4_sparse_config(): + from tensorrt_llm.llmapi.llm_args import DeepSeekV4SparseAttentionConfig + + return DeepSeekV4SparseAttentionConfig( + index_n_heads=64, + index_head_dim=128, + index_topk=512, + # Ratio-1, ratio-4 and ratio-128 layers, then the SWA-only MTP layer. + compress_ratios=[1, 4, 128, 1], + skip_indexer_for_short_seqs=False, + ) + + +@pytest.fixture(scope="module") +def deepseek_v4(): + from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import DeepseekV4TrtllmAttention + + batch_size = 4 + manager = _deepseek_v4_manager(batch_size) + draft_manager = _deepseek_v4_manager(batch_size, leading_request_ids=[batch_size]) + creation = _creation( + manager, + batch_size, + enable_context_mla_with_cached_kv=True, + sparse_metadata_params=_deepseek_v4_sparse_config().to_sparse_metadata_params(), + max_num_heads_per_kv=64, + ) + # The last request crosses a 128-token compression boundary. + cached = [HISTORY - 7 * i for i in range(batch_size - 1)] + [254] + yield _family( + DeepseekV4TrtllmAttention, + creation, + _batch(range(batch_size), cached), + num_heads=64, + extension_names=("TrtllmCoreMetadata", "DeepseekV4Metadata"), + draft_manager=draft_manager, + ) + manager.shutdown() + draft_manager.shutdown() + + +_CSA2_REQUEST_IDS = [41, 97] + + +def _csa2_manager(): + from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager + from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout + from tensorrt_llm.llmapi.llm_args import DraftTargetDecodingConfig, KvCacheConfig + + manager = CSA2CacheManager( + KvCacheConfig( + enable_block_reuse=False, + max_tokens=4096, + enable_swa_scratch_reuse=True, + dtype="fp8", + ), + CacheType.SELFKONLY, + layout=CSA2Layout((0, 2, 2, 1), (1, 3), (1, 3)), + num_layers=4, + tokens_per_block=128, + mapping=Mapping(), + max_seq_len=512, + max_batch_size=len(_CSA2_REQUEST_IDS), + max_input_len=512, + max_num_tokens=1024, + dtype=DataType.BF16, + vocab_size=1024, + spec_config=DraftTargetDecodingConfig( + max_draft_len=MAX_DRAFT_TOKENS, speculative_model="attention-test-draft" + ), + ) + manager.add_dummy_requests( + request_ids=_CSA2_REQUEST_IDS, + token_nums=[HISTORY + 1, HISTORY - 6], + is_gen=True, + max_num_draft_tokens=MAX_DRAFT_TOKENS, + ) + manager._stream.synchronize() + return manager + + +# CSA2 buffer banks, host staging and device descriptor tables: their contents +# name addresses or hold scratch, so a draft's own differ from a derived one's. +_CSA2_ADDRESS_STATE = { + "_csa2_buffers", + "_csa2_host_staging", + "_csa2_owner_descriptors", + "_csa2_swa_descriptors", + "_csa2_upload_slab", +} + + +def _csa2_state(metadata): + """The comparable state, the compression batches' tensors included.""" + + found = { + key: value + for key, value in comparable_state(metadata).items() + if (key[1][0] if isinstance(key[1], tuple) else key[1]) not in _CSA2_ADDRESS_STATE + } + for owner, batch in getattr(metadata, "_csa2_compression", {}).items(): + for name in ( + "kv_page_table", + "score_page_table", + "kv_lengths", + "start_positions", + "cu_seq_lengths", + "cu_compressed_lengths", + ): + tensor = getattr(batch, name) + found[("CSA2Metadata", ("_csa2_compression", owner, name))] = ( + "tensor", + tensor.shape, + tensor.dtype, + tensor.cpu().clone(), + ) + found[("CSA2Metadata", ("_csa2_compression", owner, "sizes"))] = ( + batch.output_rows, + batch.page_size, + batch.max_compressed_per_request, + ) + return found + + +def _csa2_resolve_swa_slots(metadata): + """The SWA slots a forward's first CSA2 layer resolves.""" + metadata._ensure_swa_slots() + + +@pytest.fixture(scope="module") +def csa2(): + from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention + + manager, draft_manager = _csa2_manager(), _csa2_manager() + yield _family( + CSA2TrtllmAttention, + _creation(manager, len(_CSA2_REQUEST_IDS)), + _batch(_CSA2_REQUEST_IDS, [HISTORY, HISTORY - 7]), + num_heads=64, + state=_csa2_state, + banks=frozenset({"_csa2_buffers"}), + forward=_csa2_resolve_swa_slots, + extension_names=("TrtllmCoreMetadata", "CSA2Metadata"), + draft_manager=draft_manager, + ) + manager.shutdown() + draft_manager.shutdown() + + +_BLACKWELL_ONLY = {"deepseek_v4", "csa2"} + + +@pytest.fixture(params=["dense", "dsa", "deepseek_v4", "csa2"]) +def family(request): + if request.param in _BLACKWELL_ONLY and getSMVersion() < 100: + pytest.skip(f"{request.param} needs Blackwell") + return request.getfixturevalue(request.param) + + +@pytest.fixture(params=["dense", "deepseek_v4", "csa2"]) +def family_with_draft_manager(request): + if request.param in _BLACKWELL_ONLY and getSMVersion() < 100: + pytest.skip(f"{request.param} needs Blackwell") + return request.getfixturevalue(request.param) + + +# --- Tests ---------------------------------------------------------------------- + + +def test_adopted_draft_equals_a_derived_draft(family): + adopted = _prepared(family, reuse=True) + derived = _prepared(family, reuse=False) + assert adopted.draft_adopts_target_state + assert not derived.draft_adopts_target_state + assert adopted.draft.kv_cache_manager is adopted.kv_cache_manager + + different = _state_differences(family.state(adopted.draft), family.state(derived.draft)) + assert different == [], different + _draft_step(family, adopted.draft, adopted) + _draft_step(family, derived.draft, derived) + different = _state_differences(family.state(adopted.draft), family.state(derived.draft)) + assert different == [], different + + +def test_loop_start_after_a_target_forward_equals_a_derived_draft(family): + """The step carries the overlap scheduler's KV-length correction + (``kv_lens_offsets``), which the target and a deriving draft apply in their + prepare and an adopting draft takes over at its loop's start.""" + batch_size = len(family.batch.request_ids) + # One draft token rejected for every request. + offsets = torch.full((batch_size,), -1, dtype=torch.int32, device="cuda") + batch = dataclasses.replace(family.batch, kv_lens_offsets=offsets) + drafts = [] + for reuse in (True, False): + metadata = _prepared(family, reuse=reuse, start_draft=False, batch=batch) + _prepare_forward(family, metadata) + draft = start_draft_loop(metadata) + assert draft is metadata.draft + torch.cuda.synchronize() + drafts.append(draft) + different = _state_differences(family.state(drafts[0]), family.state(drafts[1])) + assert different == [], different + for draft in drafts: + _prepare_forward(family, draft) + torch.cuda.synchronize() + different = _state_differences(family.state(drafts[0]), family.state(drafts[1])) + assert different == [], different + + +def test_draft_step_leaves_the_target_untouched(family): + metadata = _prepared(family, reuse=True) + before = family.state(metadata) + num_tokens = metadata.num_tokens + draft_kv_lens = metadata.draft.kv_lens_cuda.clone() + batch_size = len(family.batch.request_ids) + + _draft_step(family, metadata.draft, metadata) + + different = _state_differences(family.state(metadata), before) + assert different == [], different + assert metadata.num_tokens == num_tokens + assert metadata.draft.num_tokens == batch_size + assert torch.equal(metadata.draft.kv_lens_cuda[:batch_size], draft_kv_lens[:batch_size] + 1) + + +def test_written_buffers_keep_their_own_storage(family): + """No draft tensor shares storage with a written or private target tensor. + + A family's banks hold every buffer of their extension, the ones a draft + binds included, so they are not kept apart. + """ + metadata = _prepared(family, reuse=True) + target_extensions = _extensions_by_type(metadata) + kept_apart = set() + for extension_type, extension in _extensions_by_type(metadata.draft).items(): + source = target_extensions[extension_type] + written = extension_type._declared_union("draft_written") + private = extension_type._declared_union("draft_private") + for name in (written | private) - family.banks: + mine = dict(_tensor_items(name, vars(extension).get(name))) + for key, value in _tensor_items(name, vars(source).get(name)): + # The KV cache manager's own buffers are one object on both. + if mine.get(key) is not value: + kept_apart.add(value.untyped_storage().data_ptr()) + aliased = [ + key + for key, tensor in _extension_tensors(metadata.draft).items() + if tensor.untyped_storage().data_ptr() in kept_apart + ] + assert aliased == [], aliased + + +def test_every_attribute_of_an_adopting_extension_is_listed(family): + metadata = _prepared(family, reuse=True) + _draft_step(family, metadata.draft, metadata) + target_extensions = _extensions_by_type(metadata) + unlisted = {} + for extension_type, extension in _extensions_by_type(metadata.draft).items(): + assert extension_type.adopts_target_state, extension_type.__name__ + names = set(vars(extension)) | set(vars(target_extensions[extension_type])) + missing = extension_type.draft_unlisted(names) + if missing: + unlisted[extension_type.__name__] = sorted(missing) + assert unlisted == {}, unlisted + + +def test_draft_adoption_is_capture_safe(family): + """A draft loop's start takes the target's state again inside a CUDA graph + capture: only device copies, which a replay repeats.""" + metadata = _prepared(family, reuse=True) + expected = family.state(metadata.draft) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + start_draft_loop(metadata) + # The replay alone restores the device buffers a draft loop overwrote. + for extension_type, extension in _extensions_by_type(metadata.draft).items(): + for name in extension_type._declared_union("draft_written"): + for _, tensor in _tensor_items(name, vars(extension).get(name)): + if tensor.is_cuda: + tensor.zero_() + graph.replay() + torch.cuda.synchronize() + different = _state_differences(family.state(metadata.draft), expected) + assert different == [], different + + +def test_separate_draft_cache_reads_the_draft_manager(family_with_draft_manager): + """With a separate draft KV cache, the draft's state comes from the draft + manager: after the target's prepare and the draft loop's start, it equals a + metadata prepared over that manager directly.""" + from tensorrt_llm._torch.attention.backends.extensions import init_field_owner + from tensorrt_llm._torch.attention.backends.features.draft import _DRAFT_SETTINGS + + family = family_with_draft_manager + draft_manager = family.draft_manager + target = family.backend.Metadata(**family.creation, draft_kv_cache_manager=draft_manager) + draft_settings = { + name: value + for name, value in _DRAFT_SETTINGS.items() + if (owner := init_field_owner(name)) is not None and target.find_ext(owner) is not None + } + # A draft over its own cache stages the whole block table. + reference = family.backend.Metadata( + **{ + **family.creation, + "kv_cache_manager": draft_manager, + "one_model_draft": False, + "full_block_table_width": True, + **draft_settings, + } + ) + for metadata in (target, target.draft, reference): + _zero_buffers(metadata) + target.prepare(family.batch) + draft = start_draft_loop(target) + reference.prepare(family.batch) + torch.cuda.synchronize() + assert draft is target.draft + assert not target.draft_adopts_target_state + assert draft.kv_cache_manager is draft_manager + + def selected(metadata): + return { + key: value + for key, value in family.state(metadata).items() + if key[0] in family.extension_names + } + + different = _state_differences(selected(draft), selected(reference)) + assert different == [], different + + +def test_copies_reproduce_creation_config(family): + """Constructed copies of the target, its draft and its CUDA graph copy + report their sources' creation configuration, before and after prepare.""" + metadata = family.backend.Metadata(**family.creation) + check_copy_reproduces_creation_config(metadata) + metadata.prepare(family.batch) + torch.cuda.synchronize() + check_copy_reproduces_creation_config(metadata) + check_copy_reproduces_creation_config(metadata.draft) + graph = metadata.create_cuda_graph_metadata( + len(family.batch.request_ids), max_draft_tokens=MAX_DRAFT_TOKENS + ) + check_copy_reproduces_creation_config(graph) + + +def test_glm_tp8_dsa_draft_refresh_matches_base_dsa(dsa): + from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step import ( # noqa: E501 + GlmTp8DSAMetadata, + step_ineligibility, + ) + from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata + + base = _prepared(dsa, reuse=True) + glm = _prepared(dsa, reuse=True, extension_types=(GlmTp8DSAMetadata,)) + assert type(glm.ext(DSAMetadata)) is GlmTp8DSAMetadata + assert type(glm.draft.ext(DSAMetadata)) is GlmTp8DSAMetadata + base_dsa = base.draft.ext(DSAMetadata) + glm_dsa = glm.draft.ext(DSAMetadata) + for extension in (base_dsa, glm_dsa): + extension.skip_indexer_for_gen_reqs = False + extension.skip_indexer_for_ctx_reqs = False + assert step_ineligibility(glm.draft, glm.draft.num_seqs, glm.draft.num_contexts) is None + + _draft_step(dsa, base.draft, base) + _draft_step(dsa, glm.draft, glm) + for name in ( + "slot_mapping_fp8", + "slot_mapping_scale", + "kv_lens_cuda_2d", + "scheduler_metadata_buffer", + "scheduler_metadata_buffer_full_next_n", + "gen_kv_indptr", + "gen_cached_token_indptr", + ): + torch.testing.assert_close(getattr(glm_dsa, name), getattr(base_dsa, name), rtol=0, atol=0) + torch.testing.assert_close( + glm_dsa.index_share.req_idx_per_token, + base_dsa.index_share.req_idx_per_token, + rtol=0, + atol=0, + ) + + +def test_csa2_draft_does_not_consume_an_swa_publication_of_its_own(csa2): + from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata + + if getSMVersion() < 100: + pytest.skip("csa2 needs Blackwell") + manager = csa2.creation["kv_cache_manager"] + consumed = [] + original = type(manager).consume_swa_publication + + def consume(self): + consumed.append(self) + return original(self) + + type(manager).consume_swa_publication = consume + try: + metadata = _prepared(csa2, reuse=True) + finally: + type(manager).consume_swa_publication = original + assert metadata.draft_adopts_target_state + # Only the target's CSA2 consumes the publication its core opened. + assert consumed == [manager] + assert manager.consume_swa_publication() is None + target, draft = metadata.ext(CSA2Metadata), metadata.draft.ext(CSA2Metadata) + assert draft.csa2_global_page_tables == target.csa2_global_page_tables + assert draft.csa2_positions is not target.csa2_positions diff --git a/tests/unittest/_torch/attention/test_flashinfer_attention.py b/tests/unittest/_torch/attention/test_flashinfer_attention.py index 94e154517ebf..99b6f93b10a6 100644 --- a/tests/unittest/_torch/attention/test_flashinfer_attention.py +++ b/tests/unittest/_torch/attention/test_flashinfer_attention.py @@ -18,7 +18,7 @@ from tensorrt_llm._torch.attention.backends import \ flashinfer as flashinfer_backend from tensorrt_llm._torch.attention.backends.flashinfer import ( - FlashInferWrappers, PlanParams) + FlashInferCoreMetadata, FlashInferWrappers, PlanParams) from tensorrt_llm._torch.attention.backends.interface import \ PredefinedAttentionMask from tensorrt_llm._torch.metadata import KVCacheParams @@ -31,21 +31,27 @@ from tensorrt_llm.runtime.kv_cache_manager_v2 import BAD_PAGE_INDEX -class TestingFlashInferAttentionMetadata(FlashInferAttentionMetadata): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._num_times_planned = defaultdict(int) +class TestingFlashInferCoreMetadata(FlashInferCoreMetadata): + """Counts the plans the FlashInfer core makes.""" - def get_num_plans(self, plan_params) -> int: - return self._num_times_planned[plan_params] + def __init__(self, metadata, *, source=None, **config): + super().__init__(metadata, source=source, **config) + self.num_times_planned = defaultdict(int) def _plan_with_params(self, plan_params, flashinfer_backend: str = "fa2"): if self.needs_plan(plan_params): - self._num_times_planned[plan_params] += 1 + self.num_times_planned[plan_params] += 1 return super()._plan_with_params(plan_params, flashinfer_backend) +class TestingFlashInferAttentionMetadata(FlashInferAttentionMetadata): + + core_metadata_type = TestingFlashInferCoreMetadata + + def get_num_plans(self, plan_params) -> int: + return self.core().num_times_planned[plan_params] + + @dataclass(repr=False) class Scenario: num_layers: int @@ -86,12 +92,14 @@ def test_swa_page_sanitization_uses_local_window_order(self) -> None: sliding_page_indices = torch.tensor([BAD_PAGE_INDEX, 3], dtype=torch.int32) - metadata._sanitize_swa_page_indices(sliding_page_indices, layer_idx=4) + metadata.core()._sanitize_swa_page_indices(sliding_page_indices, + layer_idx=4) torch.testing.assert_close(sliding_page_indices, torch.tensor([0, 3], dtype=torch.int32)) full_page_indices = torch.tensor([BAD_PAGE_INDEX, 3], dtype=torch.int32) - metadata._sanitize_swa_page_indices(full_page_indices, layer_idx=5) + metadata.core()._sanitize_swa_page_indices(full_page_indices, + layer_idx=5) torch.testing.assert_close( full_page_indices, torch.tensor([BAD_PAGE_INDEX, 3], dtype=torch.int32), @@ -163,10 +171,9 @@ def get_layer_page_index_scale(self, layer_idx: int) -> int: kv_cache_manager=manager, request_ids=[7], workspace_buffer=torch.empty(1, dtype=torch.uint8, device="cuda"), - mamba_metadata=False, ) - self.assertEqual(metadata._primary_kv_layer_idx, 20) + self.assertEqual(metadata.core()._primary_kv_layer_idx, 20) manager.get_buffers.assert_called_once_with(20) metadata.prepare() @@ -203,7 +210,7 @@ def return_plan_params(plan_params, _flashinfer_backend): return plan_params with mock.patch.object( - metadata, + metadata.core(), "_plan_with_params", side_effect=return_plan_params, ): @@ -227,7 +234,7 @@ def return_plan_params(plan_params, _flashinfer_backend): flashinfer_backend="trtllm-gen", ) metadata.seq_lens = torch.tensor([1, 1], dtype=torch.int32) - metadata._uses_full_generation_page_table = True + metadata.core()._uses_full_generation_page_table = True full_page_single_token_plan = metadata.plan( num_heads=32, num_kv_heads=4, @@ -237,8 +244,8 @@ def return_plan_params(plan_params, _flashinfer_backend): attention_mask_type=AttentionMaskType.causal.value, flashinfer_backend="trtllm-gen", ) - metadata._uses_full_generation_page_table = False - metadata._is_shared_kv_draft_view = True + metadata.core()._uses_full_generation_page_table = False + metadata.core()._is_shared_kv_draft_view = True shared_draft_single_token_plan = metadata.plan( num_heads=32, num_kv_heads=4, @@ -248,7 +255,7 @@ def return_plan_params(plan_params, _flashinfer_backend): attention_mask_type=AttentionMaskType.causal.value, flashinfer_backend="trtllm-gen", ) - metadata._is_shared_kv_draft_view = False + metadata.core()._is_shared_kv_draft_view = False metadata.seq_lens = torch.tensor([6, 6], dtype=torch.int32) multi_token_plan = metadata.plan( num_heads=32, @@ -315,7 +322,7 @@ def test_cuda_graph_metadata_owns_a_private_plan_cache(self): attention_mask_type=AttentionMaskType.causal, ) eager_wrappers = FlashInferWrappers(is_planned=True) - metadata._plan_params_to_wrappers[plan_params] = eager_wrappers + metadata.core()._plan_params_to_wrappers[plan_params] = eager_wrappers graph_metadata = metadata.create_cuda_graph_metadata(2) @@ -323,10 +330,10 @@ def test_cuda_graph_metadata_owns_a_private_plan_cache(self): # batch size. That partitioning -- not the cache key -- is what keeps a graph wrapper from # being reused across batch sizes, so PlanParams need not carry the generation batch size. self.assertTrue(graph_metadata.is_cuda_graph) - self.assertIsNot(graph_metadata._plan_params_to_wrappers, - metadata._plan_params_to_wrappers) - self.assertEqual(graph_metadata._plan_params_to_wrappers, {}) - self.assertIs(metadata._plan_params_to_wrappers[plan_params], + self.assertIsNot(graph_metadata.core()._plan_params_to_wrappers, + metadata.core()._plan_params_to_wrappers) + self.assertEqual(graph_metadata.core()._plan_params_to_wrappers, {}) + self.assertIs(metadata.core()._plan_params_to_wrappers[plan_params], eager_wrappers) def test_generation_page_table_keeps_logical_positions(self): @@ -373,11 +380,11 @@ def test_generation_page_table_keeps_logical_positions(self): torch.tensor([0, 2], dtype=torch.int32, device="cuda"), ) torch.testing.assert_close( - metadata._paged_kv_last_page_len[:1], + metadata.core()._paged_kv_last_page_len[:1], torch.tensor([32], dtype=torch.int32, device="cuda"), ) torch.testing.assert_close( - metadata._logical_kv_lens[:1], + metadata.core()._logical_kv_lens[:1], torch.tensor([32], dtype=torch.int32, device="cuda"), ) torch.testing.assert_close( @@ -419,14 +426,14 @@ def test_spec_decode_offsets_update_append_and_decode_lengths(self): positions = torch.tensor([31, 32, 33, 34, 62, 63, 64, 65], dtype=torch.int32, device="cuda") - metadata._cached_token_lens[:2].copy_(cached_token_lens) - metadata._logical_kv_lens[:2].copy_(cached_token_lens + 4) - metadata._uses_full_generation_page_table = True - metadata._positions[:8].copy_(positions) + metadata.core()._cached_token_lens[:2].copy_(cached_token_lens) + metadata.core()._logical_kv_lens[:2].copy_(cached_token_lens + 4) + metadata.core()._uses_full_generation_page_table = True + metadata.core()._positions[:8].copy_(positions) kv_lens_buffer = torch.tensor([35, 66], dtype=torch.int32, device="cuda") - metadata._plan_params_to_wrappers = { + metadata.core()._plan_params_to_wrappers = { object(): FlashInferWrappers( is_planned=True, @@ -443,15 +450,15 @@ def test_spec_decode_offsets_update_append_and_decode_lengths(self): ) torch.testing.assert_close( - metadata._cached_token_lens[:2], + metadata.core()._cached_token_lens[:2], torch.tensor([28, 61], dtype=torch.int32, device="cuda"), ) torch.testing.assert_close( - metadata._logical_kv_lens[:2], + metadata.core()._logical_kv_lens[:2], torch.tensor([32, 65], dtype=torch.int32, device="cuda"), ) torch.testing.assert_close( - metadata._positions[:8], + metadata.core()._positions[:8], torch.tensor([28, 29, 30, 31, 61, 62, 63, 64], dtype=torch.int32, device="cuda"), @@ -467,11 +474,12 @@ def test_spec_decode_offsets_update_append_and_decode_lengths(self): tokens_per_generation=4, restore=True, ) - torch.testing.assert_close(metadata._cached_token_lens[:2], + torch.testing.assert_close(metadata.core()._cached_token_lens[:2], cached_token_lens) - torch.testing.assert_close(metadata._logical_kv_lens[:2], + torch.testing.assert_close(metadata.core()._logical_kv_lens[:2], cached_token_lens + 4) - torch.testing.assert_close(metadata._positions[:8], positions) + torch.testing.assert_close(metadata.core()._positions[:8], + positions) torch.testing.assert_close( kv_lens_buffer, torch.tensor([35, 66], dtype=torch.int32, device="cuda"), @@ -544,24 +552,26 @@ def create_manager(): v = torch.randn_like(q) self.assertEqual( layer.forward(q, k, v, draft_metadata).shape, q.shape) - for wrappers in draft_metadata._plan_params_to_wrappers.values(): + for wrappers in draft_metadata.core( + )._plan_params_to_wrappers.values(): torch.testing.assert_close( wrappers.decode_wrapper._kv_lens_buffer[:2], draft_metadata.kv_lens_cuda[:2], ) with mock.patch.object( - draft_metadata, + draft_metadata.core(), "_plan_with_params", - wraps=draft_metadata._plan_with_params, + wraps=draft_metadata.core()._plan_with_params, ) as replan, mock.patch.object( - draft_metadata, + draft_metadata.core(), "_build_decode_block_tables", ) as refresh_block_tables: metadata.prepare() replan.assert_not_called() - self.assertEqual(refresh_block_tables.call_count, - len(draft_metadata._plan_params_to_wrappers)) + self.assertEqual( + refresh_block_tables.call_count, + len(draft_metadata.core()._plan_params_to_wrappers)) finally: target_manager.shutdown() draft_manager.shutdown() @@ -599,7 +609,7 @@ def plan(self, **kwargs): "BatchPrefillWithRaggedKVCacheWrapper", FakeRaggedWrapper, ): - metadata._plan_ragged_no_kv(plan_params, wrapper, "cudnn") + metadata.core()._plan_ragged_no_kv(plan_params, wrapper, "cudnn") self.assertIn("kv_indptr", wrapper.plan_kwargs) self.assertNotIn("v_indptr", wrapper.plan_kwargs) @@ -831,15 +841,17 @@ def test_flashinfer_attention(self, scenario: Scenario): results_1.append(result_1) del cache_buf - for plan_params in attn_metadata._plan_params_to_wrappers.keys(): + for plan_params in attn_metadata.core( + )._plan_params_to_wrappers.keys(): self.assertEqual(attn_metadata.get_num_plans(plan_params), 1) # prepare() defers re-planning to forward_impl only when multiple # wrappers share one workspace_buffer (hybrid attention); for the # single-wrapper case it re-plans eagerly so cuda-graph capture works. attn_metadata.prepare() - defer_plan = len(attn_metadata._plan_params_to_wrappers) > 1 - for wrappers in attn_metadata._plan_params_to_wrappers.values(): + defer_plan = len(attn_metadata.core()._plan_params_to_wrappers) > 1 + for wrappers in attn_metadata.core( + )._plan_params_to_wrappers.values(): self.assertEqual(wrappers.is_planned, not defer_plan) # [context_1, gen_1] @@ -880,15 +892,17 @@ def test_flashinfer_attention(self, scenario: Scenario): context_sequence_lengths[0] + 1) results_2.append(result_2) - for plan_params in attn_metadata._plan_params_to_wrappers.keys(): + for plan_params in attn_metadata.core( + )._plan_params_to_wrappers.keys(): self.assertEqual(attn_metadata.get_num_plans(plan_params), 1) # prepare() defers re-planning to forward_impl only when multiple # wrappers share one workspace_buffer (hybrid attention); for the # single-wrapper case it re-plans eagerly so cuda-graph capture works. attn_metadata.prepare() - defer_plan = len(attn_metadata._plan_params_to_wrappers) > 1 - for wrappers in attn_metadata._plan_params_to_wrappers.values(): + defer_plan = len(attn_metadata.core()._plan_params_to_wrappers) > 1 + for wrappers in attn_metadata.core( + )._plan_params_to_wrappers.values(): self.assertEqual(wrappers.is_planned, not defer_plan) # [context_2, gen_2] @@ -928,15 +942,17 @@ def test_flashinfer_attention(self, scenario: Scenario): context_sequence_lengths[1] + 1) results_3.append(result_3) - for plan_params in attn_metadata._plan_params_to_wrappers.keys(): + for plan_params in attn_metadata.core( + )._plan_params_to_wrappers.keys(): self.assertEqual(attn_metadata.get_num_plans(plan_params), 1) # prepare() defers re-planning to forward_impl only when multiple # wrappers share one workspace_buffer (hybrid attention); for the # single-wrapper case it re-plans eagerly so cuda-graph capture works. attn_metadata.prepare() - defer_plan = len(attn_metadata._plan_params_to_wrappers) > 1 - for wrappers in attn_metadata._plan_params_to_wrappers.values(): + defer_plan = len(attn_metadata.core()._plan_params_to_wrappers) > 1 + for wrappers in attn_metadata.core( + )._plan_params_to_wrappers.values(): self.assertEqual(wrappers.is_planned, not defer_plan) # assert value @@ -1216,12 +1232,13 @@ def test_ragged_prefill_no_kv_cache_uses_cudnn_plan(self) -> None: ) self.assertEqual(attention_output.shape, (total_tokens, hidden_size)) - self.assertGreaterEqual(len(attn_metadata._plan_params_to_wrappers), 1) + self.assertGreaterEqual( + len(attn_metadata.core()._plan_params_to_wrappers), 1) # FlashInfer stores the chosen implementation on the wrapper (see # BatchPrefillWithRaggedKVCacheWrapper.__init__: self._backend = backend). # TRT-LLM no-cache ragged path passes backend="cudnn" in flashinfer.py. - for flashinfer_wrappers in attn_metadata._plan_params_to_wrappers.values( - ): + for flashinfer_wrappers in attn_metadata.core( + )._plan_params_to_wrappers.values(): ragged_wrapper = flashinfer_wrappers.ragged_prefill_wrapper self.assertIsNotNone(ragged_wrapper) self.assertEqual( @@ -1380,7 +1397,7 @@ def _plan_with_params(self, plan_params): self.num_blocks[self.num_contexts:]) wrappers.is_planned = True - refresh = FlashInferAttentionMetadata._refresh_fa2_cuda_graph_plans + refresh = FlashInferCoreMetadata._refresh_fa2_cuda_graph_plans def _refresh_wrappers(page_tuple): diff --git a/tests/unittest/_torch/attention/test_flashinfer_decode_tensor_cores.py b/tests/unittest/_torch/attention/test_flashinfer_decode_tensor_cores.py index 5a2333a73a4d..f488b4c5ee35 100644 --- a/tests/unittest/_torch/attention/test_flashinfer_decode_tensor_cores.py +++ b/tests/unittest/_torch/attention/test_flashinfer_decode_tensor_cores.py @@ -27,7 +27,7 @@ from tensorrt_llm._torch.attention.backends.flashinfer import ( FI_DECODE_TENSOR_CORES_ENV, - FlashInferAttentionMetadata, + FlashInferCoreMetadata, PlanParams, decode_tensor_cores_override, ) @@ -59,7 +59,7 @@ def use_tensor_cores( use_graph_tensor_cores: bool = False, ) -> bool: """``_use_tensor_cores`` reads only its arguments, never ``self``.""" - return FlashInferAttentionMetadata._use_tensor_cores( + return FlashInferCoreMetadata._use_tensor_cores( None, plan_params, flashinfer_backend, use_graph_tensor_cores ) diff --git a/tests/unittest/_torch/attention/test_fmha_interface.py b/tests/unittest/_torch/attention/test_fmha_interface.py index 8706cd09c6b9..4b3d1402080c 100644 --- a/tests/unittest/_torch/attention/test_fmha_interface.py +++ b/tests/unittest/_torch/attention/test_fmha_interface.py @@ -20,8 +20,10 @@ import pytest import torch +from _torch.helpers import bare_extension from fmha_test_utils import FakeAttention +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.fmha.fallback import FallbackFmha from tensorrt_llm._torch.attention.backends.fmha.interface import Fmha, FmhaPhase from tensorrt_llm._torch.attention.backends.fmha.registry import FMHA_LIBS @@ -57,7 +59,9 @@ def test_fallback_support_matches_thop_kv_update_contract( ) -> None: attn = FakeAttention() fmha = FallbackFmha(attn) - metadata = SimpleNamespace(is_cross=is_cross, helix_position_offsets=None, num_contexts=0) + metadata = SimpleNamespace( + is_cross=is_cross, num_contexts=0, find_ext=lambda extension_type: None + ) forward_args = AttentionForwardArgs(is_fused_qkv=True, update_kv_cache=update_kv_cache) assert fmha.is_supported(None, None, None, metadata, forward_args) is expected @@ -65,7 +69,7 @@ def test_fallback_support_matches_thop_kv_update_contract( def test_fallback_rejects_raw_fp8_input() -> None: attn = FakeAttention() fmha = FallbackFmha(attn) - metadata = SimpleNamespace(is_cross=False, helix_position_offsets=None, num_contexts=0) + metadata = SimpleNamespace(is_cross=False, num_contexts=0, find_ext=lambda extension_type: None) q = torch.empty((1, 128), dtype=torch.float8_e4m3fn) assert not fmha.is_supported(q, None, None, metadata, AttentionForwardArgs(is_fused_qkv=True)) @@ -76,7 +80,7 @@ def test_fallback_separate_qkv_support(mode: str) -> None: attn.is_mla_enable = mode == "mla" fmha = FallbackFmha(attn) metadata = SimpleNamespace( - is_cross=mode == "cross", helix_position_offsets=None, num_contexts=0 + is_cross=mode == "cross", num_contexts=0, find_ext=lambda extension_type: None ) forward_args = AttentionForwardArgs( is_fused_qkv=False, sage_attn_num_elts_per_blk_q=2 if mode == "sage" else 0 @@ -85,6 +89,14 @@ def test_fallback_separate_qkv_support(mode: str) -> None: assert fmha.is_supported(q, q, q, metadata, forward_args) is (mode != "ordinary") +# The metadata family with the core fields it forwards, as a mock spec. +_METADATA_SPEC = type( + "TrtllmAttentionMetadataSpec", + (TrtllmAttentionMetadata,), + dict.fromkeys(TrtllmAttentionMetadata.core_metadata_type.declared_names()), +) + + @pytest.mark.parametrize("initial_fused_qkv", [False, True]) @pytest.mark.parametrize( "input_type,sparse", @@ -124,13 +136,13 @@ def test_mla_forward_clears_fused_qkv_before_fmha_selection( attn._fmha_manager = Mock() attn._fmha_manager.select.return_value = fmha - metadata = Mock(spec=TrtllmAttentionMetadata) - metadata.token_major_gen_view.return_value = None + metadata = Mock(spec=_METADATA_SPEC) + metadata.token_major_view.return_value = None metadata.presented_token_major.return_value = nullcontext() metadata.is_cross = False - metadata.enable_flash_mla = False - metadata.spec_bl_tree_first_sparse_mask_offset_kv = None - metadata.spec_decoding_bl_tree_mask = None + extensions = {SpecDecodingMetadata: bare_extension(SpecDecodingMetadata, metadata)} + metadata.find_ext.side_effect = extensions.get + metadata.ext.side_effect = extensions.__getitem__ metadata.max_context_q_len_override = None metadata.kv_cache_manager = None lengths = torch.ones(1, dtype=torch.int32) @@ -189,7 +201,7 @@ def test_support_forwards_request_and_phase(phase: FmhaPhase | None, supported: attn = cast(TrtllmAttention, FakeAttention()) fmha = _MinimalFmha(attn) q, k, v = (torch.empty((2, 4)) for _ in range(3)) - metadata = Mock(spec=TrtllmAttentionMetadata) + metadata = Mock(spec=_METADATA_SPEC) forward_args = AttentionForwardArgs() with patch.object(fmha, "_is_supported", return_value=supported) as hook: @@ -203,7 +215,7 @@ def test_default_hooks_accept_requests() -> None: assert _MinimalFmha.is_available(attn) assert fmha.is_supported( - torch.empty((2, 4)), None, None, Mock(spec=TrtllmAttentionMetadata), AttentionForwardArgs() + torch.empty((2, 4)), None, None, Mock(spec=_METADATA_SPEC), AttentionForwardArgs() ) @@ -257,7 +269,7 @@ def test_support_checks_capabilities_before_implementation( attn = cast(TrtllmAttention, FakeAttention()) q, k, v = (torch.empty((2, 4)) for _ in range(3)) - metadata = Mock(spec=TrtllmAttentionMetadata) + metadata = Mock(spec=_METADATA_SPEC) forward_args = AttentionForwardArgs(**request_args) if requested else AttentionForwardArgs() for fmha in (_MinimalFmha(attn), _CapableFmha(attn)): diff --git a/tests/unittest/_torch/attention/test_fp4_mla.py b/tests/unittest/_torch/attention/test_fp4_mla.py index 21218c60d36b..84753fc4f83f 100644 --- a/tests/unittest/_torch/attention/test_fp4_mla.py +++ b/tests/unittest/_torch/attention/test_fp4_mla.py @@ -7,7 +7,7 @@ import pytest import torch -from _torch.helpers import triton_num_compiled +from _torch.helpers import bare_extension, stub_extension, triton_num_compiled import tensorrt_llm import tensorrt_llm._torch.attention.backends.fmha.fp4_mla as fp4_mla_fmha_backend @@ -17,6 +17,7 @@ import tensorrt_llm._torch.attention.backends.fp4_mla.metadata as fp4_mla_metadata import tensorrt_llm._torch.attention.backends.fp4_mla.v_cache as fp4_mla_v_cache import tensorrt_llm._torch.attention.backends.trtllm as trtllm_backend +from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.fmha.fp4_mla import Fp4MlaFmha from tensorrt_llm._torch.attention.backends.fp4_mla import ( FP4_BLOCK_SIZE, @@ -40,7 +41,7 @@ from tensorrt_llm._torch.attention.backends.fp4_mla.fp4_mla_context import ( _build_fp8_mla_context_metadata, ) -from tensorrt_llm._torch.attention.backends.fp4_mla.state import Fp4MlaState +from tensorrt_llm._torch.attention.backends.fp4_mla.state import Fp4MlaState, find_fp4_mla from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, AttentionInputType, @@ -70,6 +71,11 @@ _CacheType = tensorrt_llm.bindings.internal.batch_manager.CacheType +def _fp4_state(metadata=None, **values) -> Fp4MlaState: + """An FP4 MLA state of ``metadata`` with only ``values`` set, without buffers.""" + return stub_extension(Fp4MlaState, metadata, **values) + + def _swizzled_sf_offset( row_idx: int | torch.Tensor, col_idx: int | torch.Tensor, sf_per_token: int ) -> int | torch.Tensor: @@ -91,12 +97,28 @@ def _is_cutedsl_unavailable() -> bool: ) +def _attach_ext(metadata: SimpleNamespace, cls: type, extension: object) -> object: + """Serve ``extension`` as the stub's ``cls`` extension through ``find_ext``.""" + if not hasattr(metadata, "stub_extensions"): + metadata.stub_extensions = {} + metadata.find_ext = metadata.stub_extensions.get + metadata.stub_extensions[cls] = extension + return extension + + +def _fp4_mla_metadata(fp4_mla_state: object, **fields) -> SimpleNamespace: + """A metadata stub carrying ``fp4_mla_state`` as its FP4 MLA extension.""" + metadata = SimpleNamespace(**fields) + _attach_ext(metadata, Fp4MlaState, fp4_mla_state) + return metadata + + def test_fp4_mla_request_validation_uses_sparse_runtime_params() -> None: - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_sparse_topk=0, kv_cache_manager=SimpleNamespace(dtype=_DataType.NVFP4, kv_factor=1), beam_width=1, - fp4_mla_state=Fp4MlaState(hp_pool=object(), v_scale_pool=object()), + fp4_mla_state=_fp4_state(hp_pool=object(), v_scale_pool=object()), ) forward_args = AttentionForwardArgs(attention_input_type=AttentionInputType.generation_only) @@ -126,11 +148,11 @@ def _fp4_mla_validation_metadata(**overrides) -> SimpleNamespace: num_sparse_topk=0, kv_cache_manager=SimpleNamespace(dtype=_DataType.NVFP4, kv_factor=1), beam_width=1, - fp4_mla_state=Fp4MlaState(hp_pool=object(), v_scale_pool=object()), + fp4_mla_state=_fp4_state(hp_pool=object(), v_scale_pool=object()), helix_position_offsets=None, ) fields.update(overrides) - return SimpleNamespace(**fields) + return _fp4_mla_metadata(**fields) def test_fp4_mla_request_validation_still_requires_nvfp4_kv() -> None: @@ -169,7 +191,7 @@ def test_fp4_mla_missing_v_scale_pool_only_for_context_only_cache( kv_factor=1, fp4_mla_context_only_cache=context_only_cache, ), - fp4_mla_state=Fp4MlaState(hp_pool=object(), v_scale_pool=None), + fp4_mla_state=_fp4_state(hp_pool=object(), v_scale_pool=None), ) forward_args = AttentionForwardArgs(attention_input_type=attention_input_type) fmha = SimpleNamespace(attn=SimpleNamespace(sparse_params=None)) @@ -279,11 +301,11 @@ def test_fp4_mla_generation_forwards_helix_softmax_stats(monkeypatch) -> None: output = torch.empty(num_tokens, num_heads * kv_lora_rank, dtype=torch.bfloat16) softmax_stats = torch.empty(num_tokens, num_heads, 2, dtype=torch.float32) expected_stats = torch.tensor([1.25, 2.5], dtype=torch.float32).expand_as(softmax_stats) - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_generations=num_tokens, helix_position_offsets=torch.tensor([384, 769], dtype=torch.int32), _helix_spec_tokens_valid=False, - fp4_mla_state=Fp4MlaState( + fp4_mla_state=_fp4_state( generation_cache_scattered=True, prequantized_q=torch.empty(0), prequantized_q_sf=torch.empty(0), @@ -329,7 +351,7 @@ def record_decode(*args, **kwargs) -> None: assert kwargs["softmax_stats_tensor"] is softmax_stats assert softmax_stats.is_contiguous() torch.testing.assert_close(softmax_stats, expected_stats) - state = metadata.fp4_mla_state + state = find_fp4_mla(metadata) assert not state.generation_cache_scattered assert state.prequantized_q is None assert state.prequantized_q_sf is None @@ -558,15 +580,14 @@ def test_fp8_mla_context_partition_metadata_separates_q_and_kv_lengths() -> None prompt_lens_cpu = prompt_lens_cuda.clone() kv_lens_cuda = torch.tensor([128, 64], dtype=torch.int32) kv_lens_cpu = kv_lens_cuda.clone() - meta = SimpleNamespace( - num_contexts=2, - num_ctx_tokens=80, - prompt_lens_cuda_runtime=prompt_lens_cuda, - prompt_lens_cpu_runtime=prompt_lens_cpu, - host_request_types_runtime=torch.tensor([0, 0, 1], dtype=torch.int32), - fp4_mla_state=Fp4MlaState( - positions=torch.arange(80, dtype=torch.int32), fp8_context_state=object() - ), + meta = object.__new__(trtllm_backend.TrtllmAttentionMetadata) + meta.core()._num_contexts = 2 + meta.core()._num_ctx_tokens = 80 + meta.core().prompt_lens_cuda_runtime = prompt_lens_cuda + meta.core().prompt_lens_cpu_runtime = prompt_lens_cpu + meta.core().host_request_types_runtime = torch.tensor([0, 0, 1], dtype=torch.int32) + meta.replace_ext( + _fp4_state(meta, positions=torch.arange(80, dtype=torch.int32), _fp8_context_view=object()) ) scratch = SimpleNamespace( cache_manager_view=object(), @@ -588,7 +609,7 @@ def test_fp8_mla_context_partition_metadata_separates_q_and_kv_lengths() -> None assert fp8_meta.kv_lens_runtime is kv_lens_cpu assert fp8_meta.host_total_kv_lens is scratch.host_total_kv_lens assert fp8_meta.helix_position_offsets.shape == (80,) - assert fp8_meta.fp4_mla_state is None + assert fp8_meta.find_ext(Fp4MlaState) is None def _reset_triton_allocator() -> None: @@ -601,10 +622,10 @@ def _null_allocator(size: int, alignment: int, stream): def test_fp4_mla_generation_hp_page_ids_skip_mixed_batch_context_rows() -> None: - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_contexts=2, num_seqs=5, - fp4_mla_state=Fp4MlaState( + fp4_mla_state=_fp4_state( num_sequences=5, device_page_table=True, page_table_stride=3, @@ -621,13 +642,13 @@ def test_fp4_mla_generation_hp_page_ids_skip_mixed_batch_context_rows() -> None: def test_fp4_mla_cuda_graph_generation_lengths_records_capture_once(monkeypatch) -> None: corrected_kv_lens = torch.empty(2, dtype=torch.int32, device="cuda") generation_lens = torch.empty(2, dtype=torch.int32, device="cuda") - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_contexts=1, num_seqs=3, kv_lens_cuda_runtime=torch.tensor([9, 17, 25], dtype=torch.int32, device="cuda"), prompt_lens_cuda_runtime=torch.tensor([9, 1, 1], dtype=torch.int32, device="cuda"), is_cuda_graph=True, - fp4_mla_state=Fp4MlaState( + fp4_mla_state=_fp4_state( num_sequences=3, generation_kv_lens=corrected_kv_lens, generation_append_lens=generation_lens, @@ -661,7 +682,7 @@ def populate_generation_lengths(*args, **kwargs) -> None: assert args[2].data_ptr() == corrected_kv_lens.data_ptr() assert args[3].data_ptr() == generation_lens.data_ptr() assert kwargs == {"num_gen_tokens": 4, "num_gen": 2} - assert metadata.fp4_mla_state.generation_lengths_capture_recorded + assert find_fp4_mla(metadata).generation_lengths_capture_recorded assert first[0].data_ptr() == second[0].data_ptr() == corrected_kv_lens.data_ptr() assert first[1].data_ptr() == second[1].data_ptr() == generation_lens.data_ptr() @@ -687,6 +708,7 @@ def test_recompute_helix_spec_buffers_matches_elementwise_reference( metadata.mapping = SimpleNamespace(cp_size=cp_size, cp_rank=cp_rank) metadata._num_contexts = 1 metadata._num_generations = num_seqs + metadata.replace_ext(bare_extension(HelixMetadata, metadata)) untouched = 77 # The helix buffers are generation-relative; kv_lens_cuda is batch-indexed. metadata.helix_position_offsets = torch.cat( @@ -1947,7 +1969,7 @@ def _build_multi_seq_metadata( kv_global_scale = torch.tensor([FP4_MLA_KV_GLOBAL_SCALE], dtype=torch.float32, device=device) q_global_scale = torch.tensor([FP4_MLA_Q_GLOBAL_SCALE], dtype=torch.float32, device=device) - return SimpleNamespace( + return _fp4_mla_metadata( kv_cache_manager=kv_cache_manager, page_size=page_size, num_contexts=num_seqs, @@ -1958,7 +1980,7 @@ def _build_multi_seq_metadata( request_ids=request_ids, runtime_features=SimpleNamespace(has_speculative_draft_tokens=False), is_cuda_graph=False, - fp4_mla_state=Fp4MlaState( + fp4_mla_state=_fp4_state( batch_indices=batch_indices, positions=positions, _paged_kv_indices=paged_kv_indices, @@ -1988,20 +2010,19 @@ def _build_multi_seq_metadata( def _materialize_reference_cache_storage(metadata, layer_idx: int, head_dim: int) -> torch.Tensor: + fp4_mla_state = find_fp4_mla(metadata) kv_cache, sf_cache = metadata.kv_cache_manager.get_fp4_mla_cache_buffers(layer_idx) sf_cache = sf_cache.view(torch.float8_e4m3fn) storage_head_dim = kv_cache.shape[-1] * 2 static_global_scale = float(_get_fp4_mla_global_scale(metadata, kv_cache.device).item()) num_generation_sequences = metadata.num_seqs - metadata.num_contexts - page_stride = metadata.fp4_mla_state.page_table_stride - page_rows = metadata.fp4_mla_state.paged_kv_indices.view( + page_stride = fp4_mla_state.page_table_stride + page_rows = fp4_mla_state.paged_kv_indices.view( metadata.num_seqs, page_stride, ) generation_page_rows = page_rows[metadata.num_contexts : metadata.num_seqs] - indptr = ( - metadata.fp4_mla_state.paged_kv_indptr_decode[: num_generation_sequences + 1].cpu().tolist() - ) + indptr = fp4_mla_state.paged_kv_indptr_decode[: num_generation_sequences + 1].cpu().tolist() expected_indptr = [seq_idx * page_stride for seq_idx in range(num_generation_sequences + 1)] assert indptr == expected_indptr assert generation_page_rows.shape == (num_generation_sequences, page_stride) @@ -2056,13 +2077,14 @@ def _materialize_reference_cache_tokens( *, include_k_residual: bool = False, ) -> torch.Tensor: + fp4_mla_state = find_fp4_mla(metadata) kv_cache, sf_cache = metadata.kv_cache_manager.get_fp4_mla_cache_buffers(layer_idx) sf_cache = sf_cache.view(torch.float8_e4m3fn) storage_head_dim = kv_cache.shape[-1] * 2 static_global_scale = float(_get_fp4_mla_global_scale(metadata, kv_cache.device).item()) - page_rows = metadata.fp4_mla_state.paged_kv_indices.view( + page_rows = fp4_mla_state.paged_kv_indices.view( metadata.num_seqs, - metadata.fp4_mla_state.page_table_stride, + fp4_mla_state.page_table_stride, ) dequantized_pages = {} tokens = [] @@ -2121,7 +2143,7 @@ def test_fp4_mla_helix_scatter_uses_global_rope_and_local_cache_position( seed=31, ) try: - state = metadata.fp4_mla_state + state = find_fp4_mla(metadata) head_dim = kv_lora_rank + qk_rope_head_dim state.positions = torch.tensor([global_position], dtype=torch.int32, device="cuda") metadata._helix_spec_tokens_valid = False @@ -2498,11 +2520,11 @@ def _build_fp4_mla_attention_decode_case( seq_lens=seq_lens, page_size=page_size, ) - assert metadata.fp4_mla_state.v_scale_pool is not None + assert find_fp4_mla(metadata).v_scale_pool is not None persistent_pool_base = kv_cache_manager.get_mla_v_packed_pool_base() if persistent_pool_base is not None: persistent_pool_base.zero_() - metadata.fp4_mla_state.v_scale_pool.zero_() + find_fp4_mla(metadata).v_scale_pool.zero_() metadata.kv_lens_cuda_runtime = torch.tensor( context_seq_lens, @@ -2511,13 +2533,13 @@ def _build_fp4_mla_attention_decode_case( ) metadata.prompt_lens_cuda_runtime = metadata.kv_lens_cuda_runtime.clone() metadata.prompt_lens_cpu_runtime = torch.tensor(context_seq_lens, dtype=torch.int32) - metadata.fp4_mla_state.batch_indices = torch.cat( + find_fp4_mla(metadata).batch_indices = torch.cat( [ torch.full((seq_len,), seq_idx, dtype=torch.int32, device=device) for seq_idx, seq_len in enumerate(context_seq_lens) ] ) - metadata.fp4_mla_state.positions = torch.cat( + find_fp4_mla(metadata).positions = torch.cat( [torch.arange(seq_len, dtype=torch.int32, device=device) for seq_len in context_seq_lens] ) context_latent = ( @@ -2535,9 +2557,9 @@ def _build_fp4_mla_attention_decode_case( torch.cuda.synchronize() metadata.num_contexts = 0 - metadata.fp4_mla_state.num_context_blocks = 0 - metadata.fp4_mla_state.num_generation_blocks = ( - len(seq_lens) * metadata.fp4_mla_state.page_table_stride + find_fp4_mla(metadata).num_context_blocks = 0 + find_fp4_mla(metadata).num_generation_blocks = ( + len(seq_lens) * find_fp4_mla(metadata).page_table_stride ) metadata.kv_lens_cuda_runtime = torch.tensor(seq_lens, dtype=torch.int32, device=device) metadata.prompt_lens_cuda_runtime = torch.full( @@ -2547,12 +2569,12 @@ def _build_fp4_mla_attention_decode_case( (len(seq_lens),), query_len_per_seq, dtype=torch.int32 ) num_queries = len(seq_lens) * query_len_per_seq - metadata.fp4_mla_state.batch_indices = torch.arange( + find_fp4_mla(metadata).batch_indices = torch.arange( len(seq_lens), dtype=torch.int32, device=device, ).repeat_interleave(query_len_per_seq) - metadata.fp4_mla_state.positions = torch.cat( + find_fp4_mla(metadata).positions = torch.cat( [ torch.arange(context_len, seq_len, dtype=torch.int32, device=device) for context_len, seq_len in zip(context_seq_lens, seq_lens) @@ -2595,7 +2617,7 @@ def _build_fp4_mla_attention_decode_case( if enable_helix: # CP1 exercises the Helix per-token metadata contract while the # verify group below still crosses a physical page boundary. - positions = metadata.fp4_mla_state.positions + positions = find_fp4_mla(metadata).positions metadata.helix_position_offsets = positions.clone() metadata.helix_local_slots = positions.clone() metadata.helix_kv_bounds = positions + 1 @@ -2620,16 +2642,16 @@ def _build_fp4_mla_attention_decode_case( helix_is_inactive_rank=getattr(metadata, "helix_is_inactive_rank", None), ) torch.cuda.synchronize() - assert metadata.fp4_mla_state.prequantized_q is not None - assert metadata.fp4_mla_state.prequantized_q_sf is not None - assert metadata.fp4_mla_state.q_batch_capacity == num_queries + assert find_fp4_mla(metadata).prequantized_q is not None + assert find_fp4_mla(metadata).prequantized_q_sf is not None + assert find_fp4_mla(metadata).q_batch_capacity == num_queries torch.testing.assert_close(q_rope_out, q_pe, rtol=0, atol=0) canonical_generation = _materialize_reference_cache_tokens( metadata, layer_idx=0, - batch_indices=metadata.fp4_mla_state.batch_indices, - positions=metadata.fp4_mla_state.positions, + batch_indices=find_fp4_mla(metadata).batch_indices, + positions=find_fp4_mla(metadata).positions, head_dim=head_dim, ) torch.testing.assert_close( @@ -2641,7 +2663,7 @@ def _build_fp4_mla_attention_decode_case( ) # Decode reference exercises the quantized cache without an HP overlay. - metadata.fp4_mla_state.hp_pool.zero_() + find_fp4_mla(metadata).hp_pool.zero_() torch.cuda.synchronize() return kv_cache_manager, metadata, q_quant_input, kv_lora_rank, qk_rope_head_dim @@ -2656,6 +2678,7 @@ def _fp4_mla_attention_decode_reference( kv_lora_rank: int, qk_rope_head_dim: int, ) -> tuple[torch.Tensor, torch.Tensor]: + fp4_mla_state = find_fp4_mla(metadata) head_dim = kv_lora_rank + qk_rope_head_dim storage = _materialize_reference_cache_storage(metadata, 0, head_dim) dequant_cache = storage[..., :head_dim] @@ -2664,7 +2687,7 @@ def _fp4_mla_attention_decode_reference( dequant_k_residual = storage[..., head_dim : head_dim + FP4_MLA_K_RESIDUAL_DIM] num_heads = q_nope.shape[1] q_full = torch.cat((q_nope, q_pe), dim=-1).reshape(-1, head_dim) - global_scale = metadata.fp4_mla_state.q_global_scale + global_scale = fp4_mla_state.q_global_scale q_fp4, q_sf = torch.ops.trtllm.fp4_quantize_with_residual( q_full, global_scale, @@ -2681,16 +2704,16 @@ def _fp4_mla_attention_decode_reference( ) p_dequant = None - if "_fp4_mla_attention_p_buf" in metadata.fp4_mla_state.workspaces: + if "_fp4_mla_attention_p_buf" in fp4_mla_state.workspaces: p_dequant = _dequant_fp4_swizzled( - metadata.fp4_mla_state.workspaces["_fp4_mla_attention_p_buf"], - metadata.fp4_mla_state.workspaces["_fp4_mla_attention_p_sf_buf"], + fp4_mla_state.workspaces["_fp4_mla_attention_p_buf"], + fp4_mla_state.workspaces["_fp4_mla_attention_p_sf_buf"], logical_dim=metadata.page_size, sf_per_token=metadata.page_size // FP4_BLOCK_SIZE, global_scale=FP4_MLA_P_GLOBAL_SCALE, ) - indptr = metadata.fp4_mla_state.paged_kv_indptr_decode.cpu().tolist() + indptr = fp4_mla_state.paged_kv_indptr_decode.cpu().tolist() num_seqs = metadata.num_seqs - metadata.num_contexts indptr = indptr[: num_seqs + 1] kv_lens = ( @@ -2707,7 +2730,7 @@ def _fp4_mla_attention_decode_reference( for seq_idx in range(num_seqs): kv_len = kv_lens[seq_idx] page_count = indptr[seq_idx + 1] - indptr[seq_idx] - assert page_count == metadata.fp4_mla_state.page_table_stride + assert page_count == fp4_mla_state.page_table_stride assert page_count * metadata.page_size >= kv_len full_cache = dequant_cache[indptr[seq_idx] : indptr[seq_idx + 1]].reshape(-1, head_dim) assert full_cache.shape[0] >= kv_len @@ -2812,9 +2835,9 @@ def _assert_fp4_mla_attention_decode_accuracy( sm_scale=sm_scale, kv_lora_rank=kv_lora_rank, qk_rope_head_dim=qk_rope_head_dim, - prequantized_q=metadata.fp4_mla_state.prequantized_q, - prequantized_q_sf=metadata.fp4_mla_state.prequantized_q_sf, - q_batch_capacity=metadata.fp4_mla_state.q_batch_capacity, + prequantized_q=find_fp4_mla(metadata).prequantized_q, + prequantized_q_sf=find_fp4_mla(metadata).prequantized_q_sf, + q_batch_capacity=find_fp4_mla(metadata).q_batch_capacity, softmax_stats_tensor=softmax_stats, ) torch.cuda.synchronize() @@ -3168,10 +3191,10 @@ def test_fp4_mla_context_tail_uses_draft_slack_ring( ) torch.cuda.synchronize() - hp_page = int(metadata.fp4_mla_state.hp_page_indices[0].item()) + hp_page = int(find_fp4_mla(metadata).hp_page_indices[0].item()) # hp_pool is [pages, FP4 MLA layers, kv_factor, ring * head_dim]; with # MTP the draft layer adds a second FP4 layer. Layer 0 is the target. - hp_pool = metadata.fp4_mla_state.hp_pool + hp_pool = find_fp4_mla(metadata).hp_pool hp_ring = hp_pool.view(*hp_pool.shape[:3], ring_size, head_dim) torch.testing.assert_close( hp_ring[hp_page, 0, 0, expected_slot], diff --git a/tests/unittest/_torch/attention/test_fp4_mla_packed.py b/tests/unittest/_torch/attention/test_fp4_mla_packed.py index 7f2c7a13e296..f9585c03f6a5 100644 --- a/tests/unittest/_torch/attention/test_fp4_mla_packed.py +++ b/tests/unittest/_torch/attention/test_fp4_mla_packed.py @@ -13,6 +13,7 @@ import tensorrt_llm._torch.attention.backends.fp4_mla.config as fp4_mla_config import tensorrt_llm._torch.attention.backends.fp4_mla.metadata as fp4_mla_metadata import tensorrt_llm._torch.attention.backends.fp4_mla.packed_plan as fp4_mla_packed_plan +from tensorrt_llm._torch.attention.backends.fp4_mla.state import Fp4MlaState, find_fp4_mla pytestmark = pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10, @@ -20,6 +21,22 @@ ) +def _attach_ext(metadata: SimpleNamespace, cls: type, extension: object) -> object: + """Serve ``extension`` as the stub's ``cls`` extension through ``find_ext``.""" + if not hasattr(metadata, "stub_extensions"): + metadata.stub_extensions = {} + metadata.find_ext = metadata.stub_extensions.get + metadata.stub_extensions[cls] = extension + return extension + + +def _fp4_mla_metadata(fp4_mla_state: object, **fields) -> SimpleNamespace: + """A metadata stub carrying ``fp4_mla_state`` as its FP4 MLA extension.""" + metadata = SimpleNamespace(**fields) + _attach_ext(metadata, Fp4MlaState, fp4_mla_state) + return metadata + + @pytest.mark.parametrize( "num_heads,query_len,expected", [ @@ -336,7 +353,7 @@ def test_uniform_generation_lengths_fuse_the_packed_plan(monkeypatch) -> None: corrected = torch.zeros(2, dtype=torch.int32, device="cuda") generation_lens = torch.zeros(2, dtype=torch.int32, device="cuda") plan = torch.zeros(16, dtype=torch.int32, device="cuda") - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_contexts=0, num_seqs=2, kv_lens_cuda_runtime=torch.tensor([17, 25], dtype=torch.int32, device="cuda"), @@ -370,7 +387,7 @@ def test_uniform_generation_lengths_fuse_the_packed_plan(monkeypatch) -> None: uniform(metadata, 8, 2, plan_geometry=geometry) assert [name for name, _ in calls] == ["plan"] assert calls[0][1]["generation_lengths"][3] == 4 - assert metadata.fp4_mla_state.packed_plan_key == ("key",) + assert find_fp4_mla(metadata).packed_plan_key == ("key",) # The other layers of the step launch nothing. uniform(metadata, 8, 2, plan_geometry=geometry) uniform(metadata, 8, 2) @@ -380,8 +397,8 @@ def test_uniform_generation_lengths_fuse_the_packed_plan(monkeypatch) -> None: assert len(calls) == 2 and calls[1][0] == "plan" and calls[1][1]["generation_lengths"] is None # A new step whose first caller cannot plan keeps the Triton lengths kernel, # and the decode then plans alone. - metadata.fp4_mla_state.generation_lengths_num_tokens = -1 - metadata.fp4_mla_state.packed_plan_key = None + find_fp4_mla(metadata).generation_lengths_num_tokens = -1 + find_fp4_mla(metadata).packed_plan_key = None uniform(metadata, 8, 2) uniform(metadata, 8, 2, plan_geometry=geometry) assert [name for name, _ in calls[2:]] == ["lengths", "plan"] @@ -418,7 +435,7 @@ def test_packed_plan_geometry_is_shared_by_cache_update_and_decode(monkeypatch) # An indexed device, as the kernels' tensors carry: the workspace helper # compares devices exactly before reusing a buffer. device = torch.device("cuda", torch.cuda.current_device()) - metadata = SimpleNamespace( + metadata = _fp4_mla_metadata( num_contexts=0, num_seqs=4, page_size=256, @@ -438,7 +455,7 @@ def test_packed_plan_geometry_is_shared_by_cache_update_and_decode(monkeypatch) # The plan lives in the metadata workspace (a view of the stored buffer). assert ( geometry.plan.data_ptr() - == metadata.fp4_mla_state.workspaces["_fp4_mla_packed_plan"].data_ptr() + == find_fp4_mla(metadata).workspaces["_fp4_mla_packed_plan"].data_ptr() ) assert geometry.plan.numel() == _plan_mod().packed_plan_length(4, 40) again = fp4_mla_packed_plan._fp4_mla_packed_plan_geometry( diff --git a/tests/unittest/_torch/attention/test_fp4_mla_splitkv.py b/tests/unittest/_torch/attention/test_fp4_mla_splitkv.py index 8c55c71ea14d..af1b22bebed2 100644 --- a/tests/unittest/_torch/attention/test_fp4_mla_splitkv.py +++ b/tests/unittest/_torch/attention/test_fp4_mla_splitkv.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 """Split-KV reduction and per-bucket CUDA Graph workspace regressions.""" -import copy import importlib from types import ModuleType, SimpleNamespace @@ -105,8 +104,15 @@ def check() -> None: def test_splitkv_workspace_is_sized_per_graph_bucket() -> None: adapter = _module("fp4_mla_splitkv") + state_module = _module("state") heads, columns, splits, query_capacity = 128, 512, 4, 32 + def metadata_stub(is_cuda_graph: bool) -> SimpleNamespace: + state = SimpleNamespace(workspaces={}) + metadata = SimpleNamespace(is_cuda_graph=is_cuda_graph) + metadata.find_ext = lambda cls: state if cls is state_module.Fp4MlaState else None + return metadata + def partial_launch(*args, split_kv_buffers, **kwargs) -> None: # Isolate workspace ownership from the main attention kernel. These CUDA # operations are captured alongside the real adapter and real reducer. @@ -121,15 +127,11 @@ def partial_launch(*args, split_kv_buffers, **kwargs) -> None: FP4_MLA_P_GLOBAL_SCALE=1.0, run_trtllm_fp4_mla_decode_page_native_from_raw=partial_launch, ) - base_metadata = SimpleNamespace( - is_cuda_graph=False, fp4_mla_state=SimpleNamespace(workspaces={}) - ) + base_metadata = metadata_stub(is_cuda_graph=False) buckets = [] storage_pointers = set() for queries in [1, 4, 8]: - metadata = copy.copy(base_metadata) - metadata.is_cuda_graph = True - metadata.fp4_mla_state = SimpleNamespace(workspaces={}) + metadata = metadata_stub(is_cuda_graph=True) q = torch.full((queries, heads, 320), queries, dtype=torch.uint8, device="cuda") sf = torch.zeros((queries, heads, 40), dtype=torch.uint8, device="cuda") output = torch.empty((queries, heads, columns), dtype=torch.bfloat16, device="cuda") @@ -176,7 +178,7 @@ def forward( forward() workspace = { key: value - for key, value in metadata.fp4_mla_state.workspaces.items() + for key, value in state_module.find_fp4_mla(metadata).workspaces.items() if key.startswith("_fp4_mla_splitkv_") } tasks = splits * queries @@ -209,8 +211,8 @@ def forward( graph.replay() torch.testing.assert_close(output, torch.full_like(output, value), rtol=0, atol=0) for name, pointer in pointers.items(): - assert metadata.fp4_mla_state.workspaces[name].data_ptr() == pointer - assert not base_metadata.fp4_mla_state.workspaces + assert state_module.find_fp4_mla(metadata).workspaces[name].data_ptr() == pointer + assert not state_module.find_fp4_mla(base_metadata).workspaces @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) diff --git a/tests/unittest/_torch/attention/test_helix_context_slots.py b/tests/unittest/_torch/attention/test_helix_context_slots.py index 8dc6ac8dd182..ebea60afdf8e 100644 --- a/tests/unittest/_torch/attention/test_helix_context_slots.py +++ b/tests/unittest/_torch/attention/test_helix_context_slots.py @@ -7,7 +7,7 @@ import pytest import torch -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmCoreMetadata from tensorrt_llm.mapping import CpType, Mapping pytestmark = pytest.mark.cpu_only @@ -45,7 +45,7 @@ def test_helix_context_slots(cp_size: int, page: int) -> None: metadata = _metadata(cp_size, rank, page) positions = metadata.helix_position_offsets.clone() inactive = metadata.helix_is_inactive_rank.clone() - TrtllmAttentionMetadata._prepare_helix_context_slots( + TrtllmCoreMetadata._prepare_helix_context_slots( metadata, torch.tensor(prefixes + [5, 7], dtype=torch.int32) ) expected = [] @@ -86,7 +86,7 @@ def test_helix_context_slots_reset(phase: str) -> None: metadata.enable_context_mla_with_cached_kv = False else: metadata.kv_cache_manager.mapping = Mapping() - TrtllmAttentionMetadata._prepare_helix_context_slots(metadata, cached) + TrtllmCoreMetadata._prepare_helix_context_slots(metadata, cached) assert metadata.helix_context_local_slots is None @@ -100,7 +100,7 @@ def test_helix_context_slots_metadata_reuse(phase: str) -> None: inactive = metadata.helix_is_inactive_rank.clone() def check_slots(cached: torch.Tensor) -> None: - TrtllmAttentionMetadata._prepare_helix_context_slots(metadata, cached) + TrtllmCoreMetadata._prepare_helix_context_slots(metadata, cached) expected = [] for prefix, length in zip( cached[: metadata.num_contexts].tolist(), @@ -127,7 +127,7 @@ def check_slots(cached: torch.Tensor) -> None: metadata.kv_cache_manager.mapping = Mapping() else: metadata.enable_context_mla_with_cached_kv = False - TrtllmAttentionMetadata._prepare_helix_context_slots(metadata, cached) + TrtllmCoreMetadata._prepare_helix_context_slots(metadata, cached) assert metadata.helix_context_local_slots is None torch.testing.assert_close(metadata.helix_position_offsets, positions) torch.testing.assert_close(metadata.helix_is_inactive_rank, inactive) @@ -160,7 +160,7 @@ def test_helix_mla_chunk_plans(cp_size: int) -> None: chunked_loop_num=2, seq_lens_cuda=torch.empty(0), ) - TrtllmAttentionMetadata._prepare_helix_mla_chunks(metadata) + TrtllmCoreMetadata._prepare_helix_mla_chunks(metadata) assert len(metadata.helix_mla_chunk_plans) == 2 for loop, plan in enumerate(metadata.helix_mla_chunk_plans): expected_cu, expected_starts = [], [] @@ -185,5 +185,5 @@ def test_helix_mla_chunk_plans(cp_size: int) -> None: assert plan.rank_sizes == sizes.tolist() assert plan.max_local_seq_len == (cu[-1, 1:] - cu[-1, :-1]).max().item() metadata.helix_context_local_slots = None - TrtllmAttentionMetadata._prepare_helix_mla_chunks(metadata) + TrtllmCoreMetadata._prepare_helix_mla_chunks(metadata) assert metadata.helix_mla_chunk_plans is None diff --git a/tests/unittest/_torch/attention/test_metadata_creation_config.py b/tests/unittest/_torch/attention/test_metadata_creation_config.py new file mode 100644 index 000000000000..08f68140531b --- /dev/null +++ b/tests/unittest/_torch/attention/test_metadata_creation_config.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A constructed copy of a metadata reports the creation configuration of its source. + +Covers the core families and the feature extensions that attach without a +family-specific KV cache manager; the sparse tests check their own metadata +(see ``utils.metadata_contract.CREATION_CONFIG_COVERAGE``). +""" + +import pytest +import torch +from utils.metadata_contract import check_copy_reproduces_creation_config + +import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features import ( + FlashMlaMetadata, + HelixMetadata, + RecurrentConfig, + RecurrentMetadata, +) +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig +from tensorrt_llm._torch.attention.backends.flashinfer import FlashInferAttentionMetadata +from tensorrt_llm._torch.attention.backends.interface import ( + AttentionMetadata, + AttentionRuntimeFeatures, + BatchInputs, +) +from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( + MiniMaxM3MsaMetadata, +) +from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.triton_metadata import ( + MiniMaxM3TritonMetadata, +) +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata +from tensorrt_llm._torch.metadata import KVCacheParams +from tensorrt_llm._torch.models.modeling_qwen4_exp import Qwen4ExpPleState +from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager +from tensorrt_llm.bindings import DataType +from tensorrt_llm.llmapi.llm_args import KvCacheConfig +from tensorrt_llm.mapping import Mapping + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +BATCH_SIZE = 4 +TOKENS_PER_BLOCK = 32 +HISTORY = 100 +MAX_DRAFT_TOKENS = 1 + + +@pytest.fixture(scope="module") +def manager(): + manager = KVCacheManager( + KvCacheConfig(max_tokens=BATCH_SIZE * 8 * TOKENS_PER_BLOCK, enable_block_reuse=False), + tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, + num_layers=1, + num_kv_heads=2, + head_dim=64, + tokens_per_block=TOKENS_PER_BLOCK, + max_seq_len=512, + max_batch_size=BATCH_SIZE, + mapping=Mapping(world_size=1, tp_size=1, rank=0), + dtype=DataType.BF16, + ) + manager.add_dummy_requests( + request_ids=list(range(BATCH_SIZE)), token_nums=[HISTORY + 2] * BATCH_SIZE, is_gen=False + ) + yield manager + manager.shutdown() + + +def _batch(manager, query_len: int = 1 + MAX_DRAFT_TOKENS) -> BatchInputs: + return BatchInputs( + seq_lens=torch.full((BATCH_SIZE,), query_len, dtype=torch.int), + num_contexts=0, + request_ids=list(range(BATCH_SIZE)), + prompt_lens=[HISTORY] * BATCH_SIZE, + kv_cache_params=KVCacheParams( + use_cache=True, num_cached_tokens_per_seq=[HISTORY] * BATCH_SIZE + ), + ) + + +def test_trtllm_target_draft_and_graph_copies(manager): + metadata = TrtllmAttentionMetadata( + max_num_requests=BATCH_SIZE, + max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, + kv_cache_manager=manager, + runtime_features=AttentionRuntimeFeatures(has_speculative_draft_tokens=True), + spec_decoding=SpecDecodingConfig( + max_draft_len=MAX_DRAFT_TOKENS, max_total_draft_tokens=MAX_DRAFT_TOKENS + ), + one_model_draft=True, + ) + check_copy_reproduces_creation_config(metadata) + metadata.prepare(_batch(manager)) + check_copy_reproduces_creation_config(metadata) + check_copy_reproduces_creation_config(metadata.draft) + graph = metadata.create_cuda_graph_metadata(BATCH_SIZE, max_draft_tokens=MAX_DRAFT_TOKENS) + check_copy_reproduces_creation_config(graph) + graph.prepare(_batch(manager)) + check_copy_reproduces_creation_config(graph) + + +@pytest.mark.parametrize( + "extension_type", + [ + HelixMetadata, + FlashMlaMetadata, + MTPIndexShareExtension, + Qwen4ExpPleState, + MiniMaxM3MsaMetadata, + MiniMaxM3TritonMetadata, + ], + ids=lambda extension_type: extension_type.__name__, +) +def test_trtllm_feature_extensions(manager, extension_type): + metadata = TrtllmAttentionMetadata( + max_num_requests=BATCH_SIZE, + max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, + kv_cache_manager=manager, + extension_types=(extension_type,), + ) + check_copy_reproduces_creation_config(metadata) + metadata.prepare(_batch(manager, query_len=1)) + check_copy_reproduces_creation_config(metadata) + + +def test_vanilla_core(manager): + metadata = VanillaAttentionMetadata( + max_num_requests=BATCH_SIZE, + max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, + kv_cache_manager=manager, + ) + check_copy_reproduces_creation_config(metadata) + metadata.prepare(_batch(manager)) + check_copy_reproduces_creation_config(metadata) + + +def test_flashinfer_core(manager): + metadata = FlashInferAttentionMetadata( + max_num_requests=BATCH_SIZE, + max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, + kv_cache_manager=manager, + ) + check_copy_reproduces_creation_config(metadata) + metadata.prepare(_batch(manager)) + check_copy_reproduces_creation_config(metadata) + graph = metadata.create_cuda_graph_metadata(BATCH_SIZE) + check_copy_reproduces_creation_config(graph) + + +def test_recurrent_extension(): + metadata = AttentionMetadata( + max_num_requests=BATCH_SIZE, + max_num_tokens=BATCH_SIZE, + extension_types=(RecurrentMetadata,), + recurrent_config=RecurrentConfig(chunk_size=64), + ) + check_copy_reproduces_creation_config(metadata) + + +def test_cross_attention_decoder_and_sub_metadata(): + decoder = AttentionMetadata(max_num_requests=BATCH_SIZE, max_num_tokens=BATCH_SIZE) + decoder.prepare( + BatchInputs( + seq_lens=torch.tensor([3, 1], dtype=torch.int32), + num_contexts=1, + request_ids=[0, 1], + prompt_lens=[3, 5], + ) + ) + cross = decoder.create_cross_metadata([7, 0]) + check_copy_reproduces_creation_config(decoder) + check_copy_reproduces_creation_config(cross) diff --git a/tests/unittest/_torch/attention/test_mixed_precision_backends.py b/tests/unittest/_torch/attention/test_mixed_precision_backends.py index c93526bdaa89..05e7a106a56f 100644 --- a/tests/unittest/_torch/attention/test_mixed_precision_backends.py +++ b/tests/unittest/_torch/attention/test_mixed_precision_backends.py @@ -8,14 +8,16 @@ import pytest import torch +from _torch.helpers import bare_extension, refresh_from_lengths from tensorrt_llm._torch.attention.attention import Attention from tensorrt_llm._torch.attention.backends.flashinfer import FlashInferAttention from tensorrt_llm._torch.attention.backends.interface import ( AttentionRuntimeFeatures, + BatchInputs, PredefinedAttentionMask, ) -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm._torch.memory_buffer_utils import Buffers from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 @@ -550,47 +552,37 @@ def test_one_model_draft_kv_cache_dtype(is_mtp): @pytest.mark.cpu_only -def test_dsa_draft_forward_uses_draft_mixed_precision_high_table(): - """A mixed DSA draft manager's MTP layers read its own mixed-precision high page table.""" - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, +def test_staging_follows_this_metadatas_device_kv_lengths(): + """A draft's length changes reach its staging copy; another manager's staging is hidden.""" + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmCoreMetadata + + core = bare_extension(TrtllmCoreMetadata, Mock()) + manager = Mock(name="draft_kv_cache_manager") + core.kv_cache_manager = manager + core._seq_lens_cuda = torch.tensor([1, 2], dtype=torch.int32) + core.kv_lens_cuda = torch.tensor([10, 20, 0], dtype=torch.int32) + view = SimpleNamespace( + manager=manager, lengths=torch.zeros((2, 2), dtype=torch.int32), pools={0: None} ) - - metadata = Mock() - metadata.kv_cache_manager = Mock(name="draft_kv_cache_manager") - metadata.draft_replay_swapped_attrs = {} - metadata.record_draft_swap = partial(TrtllmAttentionMetadata.record_draft_swap, metadata) - metadata.swap_for_draft = partial(TrtllmAttentionMetadata.swap_for_draft, metadata) - metadata.restore_draft_swaps = partial(TrtllmAttentionMetadata.restore_draft_swaps, metadata) - target, draft = (torch.full((2, 4), -1, dtype=torch.int32) for _ in range(2)) - metadata.mixed_precision_high_block_table = target - metadata.draft_mixed_precision_high_block_table = draft - - DSAtrtllmAttentionMetadata.prepare_for_draft_forward(metadata) - assert metadata.mixed_precision_high_block_table is draft - metadata.restore_draft_swaps() - assert metadata.mixed_precision_high_block_table is target - - -@pytest.mark.cpu_only -def test_draft_forward_swaps_in_draft_staging(): - """A mixed MTP draft manager's layers run on its own staging copy during the draft forward.""" - metadata = Mock() - metadata.draft_replay_swapped_attrs = {} - metadata.record_draft_swap = partial(TrtllmAttentionMetadata.record_draft_swap, metadata) - metadata.swap_for_draft = partial(TrtllmAttentionMetadata.swap_for_draft, metadata) - metadata.restore_draft_swaps = partial(TrtllmAttentionMetadata.restore_draft_swaps, metadata) - metadata.sync_draft_mixed_precision_staging = partial( - TrtllmAttentionMetadata.sync_draft_mixed_precision_staging, metadata - ) - target, draft = Mock(name="target_staging"), Mock(name="draft_staging") - metadata.mixed_precision_staging = target - metadata.draft_mixed_precision_staging = draft - - TrtllmAttentionMetadata.prepare_for_draft_forward(metadata) - assert metadata.mixed_precision_staging is draft - metadata.restore_draft_swaps() - assert metadata.mixed_precision_staging is target + core.mixed_precision_staging = view + assert core._mixed_precision_staging_name() == "mixed" + # A draft metadata is created with a staging name of its own. + core.mixed_precision_staging_name = "draft_mixed" + assert core._mixed_precision_staging_name() == "draft_mixed" + + # A draft step advanced the device KV lengths, then the draft loop calls before_forward(). + core.kv_lens_cuda[:2] += 1 + core.resync_length_derived_state() + assert view.lengths.tolist() == [[10, 19], [11, 21]] + assert core.get_mixed_precision_kv_view(0) is view + assert core.get_mixed_precision_kv_view(1) is None + + # A staging copy of another manager is neither used nor updated. + view.manager = Mock(name="target_kv_cache_manager") + core.kv_lens_cuda[:2] += 1 + core.resync_length_derived_state() + assert view.lengths.tolist() == [[10, 19], [11, 21]] + assert core.get_mixed_precision_kv_view(0) is None @pytest.mark.cpu_only @@ -739,7 +731,7 @@ def make_metadata(): "backend", (TrtllmAttention, FlashInferAttention), ids=("trtllm", "flashinfer") ) def test_staging_lengths_follow_corrected_kv_lens(backend): - """The overlap scheduler corrects KV lengths on the device after prepare(); staging follows.""" + """The overlap scheduler corrects KV lengths on the device; staging follows.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] not in (10, 12): pytest.skip("Mixed-precision NVFP4 KV requires Blackwell") block, cached = 128, 32 @@ -754,49 +746,43 @@ def test_staging_lengths_follow_corrected_kv_lens(backend): view = metadata.mixed_precision_staging assert view.lengths.tolist() == [[cached], [cached + block]] offsets = torch.full((1,), -2, dtype=torch.int32, device="cuda") + corrected = [[cached - 2], [cached + block - 2]] + if backend is TrtllmAttention: + # prepare applies the step's device corrections to the target and its draft. + metadata.prepare(_small_inputs(manager, block, cached, kv_lens_offsets=offsets)) + view = metadata.mixed_precision_staging + assert view.lengths.tolist() == corrected + # The draft runs on its own staging copy of the draft manager. + draft_view = metadata.draft.mixed_precision_staging + assert draft_view is not view + assert draft_view.manager is draft + assert draft_view.name == "draft_mixed" + assert draft_view.lengths.tolist() == corrected + assert metadata.get_mixed_precision_kv_view(0) is view + assert metadata.draft.get_mixed_precision_kv_view(0) is draft_view + return def correct(): - if backend is TrtllmAttention: - metadata.kv_lens_cuda[:1] += offsets - metadata.on_update_kv_lens() - else: - metadata._uses_full_generation_page_table = True - metadata.apply_spec_decode_kv_lens_offsets( - offsets, num_generations=0, tokens_per_generation=1, num_chunked_contexts=1 - ) + metadata.core()._uses_full_generation_page_table = True + metadata.apply_spec_decode_kv_lens_offsets( + offsets, num_generations=0, tokens_per_generation=1, num_chunked_contexts=1 + ) correct() - corrected = [[cached - 2], [cached + block - 2]] assert view.lengths.tolist() == corrected - # The draft forward runs on the draft staging copy with the corrected lengths. - if backend is TrtllmAttention: - metadata.prepare_for_draft_forward() - assert metadata.mixed_precision_staging is metadata.draft_mixed_precision_staging - assert metadata.mixed_precision_staging.lengths.tolist() == corrected - metadata.restore_draft_swaps() - assert metadata.mixed_precision_staging is view - else: - # The first draft view sync follows the correction; later syncs precede it. - draft_view = metadata.get_draft_metadata(draft) - assert draft_view.mixed_precision_staging is metadata.draft_mixed_precision_staging - assert draft_view.mixed_precision_staging.lengths.tolist() == corrected - metadata.prepare() - correct() - assert view.lengths.tolist() == corrected - # Every draft forward starts here; the view reuses the same staging copy and - # builds none of its own. - assert metadata.get_draft_metadata(draft) is draft_view - assert draft_view.mixed_precision_staging is metadata.draft_mixed_precision_staging - assert draft_view.draft_mixed_precision_staging is None - assert draft_view.mixed_precision_staging.lengths.tolist() == corrected - # Draft layers run on another manager through this metadata: the view is - # hidden and the staging lengths are left alone. - metadata.kv_cache_manager = object() - assert metadata.get_mixed_precision_kv_view(0) is None + # The separate draft view runs on its own staging copy of the draft + # manager, which takes the corrected lengths each time the view is fetched. + draft_view = metadata.get_draft_metadata(draft) + draft_staging = draft_view.mixed_precision_staging + assert draft_staging is not view and draft_staging.manager is draft + assert draft_staging.name == "draft_mixed" + assert draft_staging.lengths.tolist() == corrected + metadata.prepare() correct() - assert view.lengths.tolist() == [[cached - 2], [cached + block - 2]] - metadata.kv_cache_manager = manager - assert metadata.get_mixed_precision_kv_view(0) is view + assert view.lengths.tolist() == corrected + assert metadata.get_draft_metadata(draft) is draft_view + assert draft_view.mixed_precision_staging is draft_staging + assert draft_staging.lengths.tolist() == corrected finally: manager.shutdown() draft.shutdown() @@ -823,7 +809,37 @@ def _small_manager(block: int, mapping: Mapping, dtype: str = "mixed_nvfp4_bfloa ) +def _small_inputs(manager, block: int, cached: int, *, seq_lens=None, kv_lens_offsets=None): + """The step _small_metadata describes, as BatchInputs (one context request).""" + return BatchInputs( + seq_lens=torch.tensor([block], dtype=torch.int32) if seq_lens is None else seq_lens, + num_contexts=1 if seq_lens is None else 0, + request_ids=[0], + prompt_lens=[cached + block], + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[cached]), + kv_lens_offsets=kv_lens_offsets, + ) + + def _small_metadata(backend, manager, mapping: Mapping, block: int, cached: int, draft=None): + if draft is not None and backend is TrtllmAttention: + # One-model speculative decoding: the draft forwards get md.draft. + return backend.Metadata( + seq_lens=torch.tensor([block], dtype=torch.int32), + num_contexts=1, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[cached]), + max_num_requests=2, + max_num_tokens=block, + max_num_sequences=2, + kv_cache_manager=manager, + draft_kv_cache_manager=draft, + one_model_draft=True, + request_ids=[0], + prompt_lens=[cached + block], + kv_layout="NHD", + mapping=mapping, + runtime_features=AttentionRuntimeFeatures(chunked_prefill=True, cache_reuse=True), + ) return backend.Metadata( seq_lens=torch.tensor([block], dtype=torch.int32), num_contexts=1, @@ -876,38 +892,6 @@ def test_mixed_precision_llm_args_accept_supported_features(overrides): ) -@pytest.mark.parametrize("backend", ["dsa", "deepseek_v4"]) -@pytest.mark.cpu_only -def test_draft_forward_hides_mixed_precision_high_rows(backend): - """A draft manager without a mixed-precision high pool skips its writes and reads.""" - from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( - DeepseekV4TrtllmAttentionMetadata, - ) - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAtrtllmAttentionMetadata, - ) - - cls = {"dsa": DSAtrtllmAttentionMetadata, "deepseek_v4": DeepseekV4TrtllmAttentionMetadata}[ - backend - ] - metadata = Mock() - metadata.kv_cache_manager = Mock(name="draft_kv_cache_manager") - metadata.draft_sliding_block_tables = None - metadata.draft_mixed_precision_high_block_table = None - metadata.draft_replay_swapped_attrs = {} - metadata.record_draft_swap = partial(TrtllmAttentionMetadata.record_draft_swap, metadata) - metadata.swap_for_draft = partial(TrtllmAttentionMetadata.swap_for_draft, metadata) - metadata.restore_draft_swaps = partial(TrtllmAttentionMetadata.restore_draft_swaps, metadata) - table = torch.full((2, 4), -1, dtype=torch.int32) - metadata.mixed_precision_high_block_table = table - - assert cls.prepare_for_draft_forward(metadata) is None - assert cls.has_mixed_precision_high_rows.fget(metadata) is False - metadata.restore_draft_swaps() - assert metadata.mixed_precision_high_block_table is table - assert cls.has_mixed_precision_high_rows.fget(metadata) is True - - @pytest.mark.cpu_only def test_gqa_decode_skips_the_kv_append_plan_of_a_mixed_layer(): """A mixed-precision layer appends into its staging copy, not through a GQA-decode plan.""" @@ -975,7 +959,7 @@ def test_staging_page_tables_follow_each_layer_page_scale(): def test_draft_staging_follows_corrected_kv_lens_in_cuda_graph(): - """Inside a CUDA graph, each draft forward's staging copy follows the corrected KV lengths.""" + """Each prepare corrects both staging copies; a captured draft step keeps the draft's in step.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] not in (10, 12): pytest.skip("Mixed-precision NVFP4 KV requires Blackwell") block, cached = 128, 32 @@ -987,23 +971,29 @@ def test_draft_staging_follows_corrected_kv_lens_in_cuda_graph(): # A separate MTP draft manager holds the same tokens; graph metadata is a one-token decode. metadata = _small_metadata(TrtllmAttention, manager, mapping, block, cached, draft=draft) metadata = metadata.create_cuda_graph_metadata(1, buffers=Buffers()) - metadata.prepare() offsets = torch.zeros(1, dtype=torch.int32, device="cuda") + decode = torch.ones(1, dtype=torch.int32) + metadata.prepare( + _small_inputs(manager, block, cached, seq_lens=decode, kv_lens_offsets=offsets) + ) + draft_core = metadata.draft.core() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): - # The overlap scheduler corrects KV lengths on the device, then a draft forward runs. - metadata.kv_lens_cuda[:1] += offsets - metadata.on_update_kv_lens() - metadata.prepare_for_draft_forward() - metadata.restore_draft_swaps() + # A draft step advances the draft's KV lengths, then readies its next forward. + draft_core.adjust_kv_lens(slice(0, 1), 1) + refresh_from_lengths(metadata.draft) for offset in (-1, -3): - # Each step, prepare() writes uncorrected lengths; the replay must correct both copies. - metadata.prepare() offsets.fill_(offset) - graph.replay() + metadata.prepare( + _small_inputs(manager, block, cached, seq_lens=decode, kv_lens_offsets=offsets) + ) corrected = [[cached + offset], [cached + 1 + offset]] assert metadata.mixed_precision_staging.lengths.tolist() == corrected - assert metadata.draft_mixed_precision_staging.lengths.tolist() == corrected + assert metadata.draft.mixed_precision_staging.lengths.tolist() == corrected + graph.replay() + advanced = [[cached + 1 + offset], [cached + 2 + offset]] + assert metadata.draft.mixed_precision_staging.lengths.tolist() == advanced + assert metadata.mixed_precision_staging.lengths.tolist() == corrected finally: manager.shutdown() draft.shutdown() @@ -1053,3 +1043,79 @@ def test_steady_decode_step_skips_mixed_precision(dtype): assert metadata.supports_steady_decode_step == (dtype == "auto") finally: manager.shutdown() + + +def _high_table_manager(high_layers): + """A KV cache manager stub with mixed-precision high layers that records page-table fills.""" + manager = SimpleNamespace( + mixed_precision_high_layers=high_layers, max_blocks_per_seq=4, filled=[] + ) + manager._fill_page_table = lambda layer, request_ids, table: manager.filled.append( + (layer, list(request_ids)) + ) + return manager + + +def _dsa_over(manager): + """The DSA mixed-precision high-table state of a metadata over ``manager``.""" + metadata = SimpleNamespace( + kv_cache_manager=manager, + max_num_sequences=2, + cuda_graph_buffers=None, + num_seqs=2, + request_ids=[3, 4], + get_empty=lambda buffers, shape, cache_name, dtype, capture_graph: torch.full( + shape, -1, dtype=dtype + ), + ) + return SimpleNamespace(metadata=metadata, mixed_precision_high_block_table=None) + + +@pytest.mark.cpu_only +def test_dsa_draft_over_a_mixed_draft_manager_fills_its_own_high_table(): + """A draft metadata over a separate mixed-precision draft manager fills its own + mixed-precision high page table from that manager; the target's stays the target's.""" + from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata + + target_manager, draft_manager = _high_table_manager({0: 7}), _high_table_manager({0: 9}) + target, draft = _dsa_over(target_manager), _dsa_over(draft_manager) + for dsa in (target, draft): + DSAMetadata._allocate_mixed_precision_high_block_table(dsa, capture_graph=False) + dsa._fill_mixed_precision_high_block_table = partial( + DSAMetadata._fill_mixed_precision_high_block_table, dsa + ) + DSAMetadata._fill_mixed_precision_high_block_tables(dsa) + assert target.mixed_precision_high_block_table is not draft.mixed_precision_high_block_table + assert target_manager.filled == [(7, [3, 4])] + assert draft_manager.filled == [(9, [3, 4])] + assert DSAMetadata.has_mixed_precision_high_rows.fget(draft) is True + + +@pytest.mark.parametrize("backend", ["dsa", "deepseek_v4"]) +@pytest.mark.cpu_only +def test_draft_without_a_high_pool_has_no_mixed_precision_high_rows(backend): + """A draft metadata whose manager has no mixed-precision high pool (DSA), or whose + MTP layers have none (DeepSeek-V4, created with builds_mixed_precision_high_tables + off), skips the high-pool writes and reads.""" + from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( + DeepseekV4Metadata, + ) + from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata + + if backend == "dsa": + draft = _dsa_over(_high_table_manager({})) + DSAMetadata._allocate_mixed_precision_high_block_table(draft, capture_graph=False) + assert draft.mixed_precision_high_block_table is None + assert DSAMetadata.has_mixed_precision_high_rows.fget(draft) is False + DSAMetadata._fill_mixed_precision_high_block_tables(draft) + assert draft.metadata.kv_cache_manager.filled == [] + else: + table = {4: torch.full((2, 4), -1, dtype=torch.int32)} + draft = SimpleNamespace( + mixed_precision_high_block_table=table, builds_mixed_precision_high_tables=False + ) + assert DeepseekV4Metadata.has_mixed_precision_high_rows.fget(draft) is False + target = SimpleNamespace( + mixed_precision_high_block_table=table, builds_mixed_precision_high_tables=True + ) + assert DeepseekV4Metadata.has_mixed_precision_high_rows.fget(target) is True diff --git a/tests/unittest/_torch/attention/test_pixtral_head104_attention.py b/tests/unittest/_torch/attention/test_pixtral_head104_attention.py index d042d98c7896..330b33860988 100644 --- a/tests/unittest/_torch/attention/test_pixtral_head104_attention.py +++ b/tests/unittest/_torch/attention/test_pixtral_head104_attention.py @@ -8,6 +8,7 @@ from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, + BatchInputs, PredefinedAttentionMask, ) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata @@ -26,14 +27,19 @@ def test_pixtral_head104_attention(dtype: torch.dtype, monkeypatch: pytest.Monke lengths = [2048, 32] num_heads, head_dim = 4, 104 metadata = TrtllmAttentionMetadata( - max_num_requests=4096, max_num_tokens=16384, kv_cache_manager=None + max_num_requests=4096, + max_num_tokens=16384, + kv_cache_manager=None, + max_seq_len=max(lengths), + ) + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor(lengths, dtype=torch.int), + num_contexts=len(lengths), + request_ids=list(range(len(lengths))), + prompt_lens=lengths, + ) ) - metadata.num_contexts = len(lengths) - metadata.request_ids = list(range(len(lengths))) - metadata.prompt_lens = lengths - metadata.seq_lens = torch.tensor(lengths, dtype=torch.int) - metadata.max_seq_len = max(lengths) - metadata.prepare() attention = TrtllmAttention( layer_idx=0, num_heads=num_heads, num_kv_heads=num_heads, head_dim=head_dim ) diff --git a/tests/unittest/_torch/attention/test_trtllm_paged_context_fmha_gate.py b/tests/unittest/_torch/attention/test_trtllm_paged_context_fmha_gate.py index f19046db2667..78b1f684c6c0 100644 --- a/tests/unittest/_torch/attention/test_trtllm_paged_context_fmha_gate.py +++ b/tests/unittest/_torch/attention/test_trtllm_paged_context_fmha_gate.py @@ -33,7 +33,10 @@ AttentionRuntimeFeatures, PredefinedAttentionMask, ) -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.trtllm import ( + TrtllmAttentionMetadata, + TrtllmCoreMetadata, +) from tensorrt_llm._utils import get_sm_version from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal import thop @@ -57,7 +60,7 @@ def _make_metadata(head_dim, features, kv_dtype=_ABSENT, tokens_per_block=32): manager.dtype = kv_dtype if tokens_per_block is not _ABSENT: manager.tokens_per_block = tokens_per_block - with mock.patch.object(TrtllmAttentionMetadata, "_post_init_with_buffers"): + with mock.patch.object(TrtllmCoreMetadata, "_allocate_buffers"): return TrtllmAttentionMetadata( max_num_requests=4, max_num_tokens=1024, @@ -463,7 +466,7 @@ def test_native_lookup_reports_absent_for_an_unbuilt_head_size(): def test_unknown_head_dim_is_not_guessed(): """A manager that exposes no head_dim (or no manager at all) cannot be judged; the gate must not block it on a guess.""" - with mock.patch.object(TrtllmAttentionMetadata, "_post_init_with_buffers"): + with mock.patch.object(TrtllmCoreMetadata, "_allocate_buffers"): metadata = TrtllmAttentionMetadata( max_num_requests=4, max_num_tokens=1024, diff --git a/tests/unittest/_torch/helpers.py b/tests/unittest/_torch/helpers.py index badb773acdda..1b4b76aace72 100644 --- a/tests/unittest/_torch/helpers.py +++ b/tests/unittest/_torch/helpers.py @@ -349,6 +349,127 @@ def gumbel_max_reference(logits: torch.Tensor, temperatures: torch.Tensor, logits.float().argmax(-1)).to(torch.int32) +def bare_extension(extension_type, metadata, **values): + """An extension of ``metadata``, constructed but not attached. + + For kernel and op tests that give a stub metadata the few fields an op + reads: ``values`` that are creation arguments are passed to the + constructor, the others are set afterwards; attach it with + ``metadata.replace_ext``. Over a metadata made without ``__init__`` the + core has no capacities, so the buffers it sizes are empty. Over a stub + that is no ``AttentionMetadata`` (or None) nothing is constructed: the + extension holds its creation defaults, its dataclass field defaults and + ``values`` only. + """ + import copy + import dataclasses + + from tensorrt_llm._torch.attention.backends.interface import \ + AttentionMetadata + + init_names = set(extension_type.declared_init_fields()) + if not isinstance(metadata, AttentionMetadata): + extension = extension_type.__new__(extension_type) + extension.metadata = metadata + for name, default in extension_type.init_field_defaults().items(): + setattr(extension, name, copy.copy(default)) + if dataclasses.is_dataclass(extension_type): + for field in dataclasses.fields(extension_type): + if field.name == "metadata": + continue + if field.default_factory is not dataclasses.MISSING: + setattr(extension, field.name, field.default_factory()) + elif field.default is not dataclasses.MISSING: + setattr(extension, field.name, field.default) + for name, value in values.items(): + setattr(extension, name, value) + return extension + config = { + name: value + for name, value in values.items() if name in init_names + } + extension = extension_type(metadata, **config) + for name, value in values.items(): + if name not in init_names: + setattr(extension, name, value) + return extension + + +def stub_extension(extension_type, metadata, **values): + """An extension of ``metadata`` that is not constructed, for ``replace_ext``. + + For tests of an extension whose constructor needs a real KV cache manager + or its sibling extensions: it holds its creation defaults, its dataclass + field defaults and ``values`` only. ``construct_extension`` constructs it + later, once it is attached and the metadata has what the constructor reads. + """ + extension = bare_extension(extension_type, None, **values) + extension.metadata = metadata + return extension + + +def construct_extension(extension, **config): + """Run ``extension``'s constructor in place, as its metadata would. + + For tests that set up an extension (``bare_extension``, or an attached + one) before it allocates: its creation values are kept unless ``config`` + replaces them, and instance attributes the constructor does not set (such + as mocked methods) stay. + """ + values = { + name: extension.creation_value(name) + for name in type(extension).declared_init_fields() + if name in vars(extension) or hasattr(type(extension), name) + } + values.update(config) + type(extension).__init__(extension, extension.metadata, **values) + return extension + + +def start_draft_loop(metadata): + """Enter ``metadata.draft_loop()`` and return the draft, as a spec worker does. + + The loop stays open for the rest of the test (until the next start). + """ + return metadata.draft_loop().__enter__() + + +def refresh_from_lengths(metadata): + """Refresh ``metadata``'s length-derived state as a draft refresh does. + + For a draft metadata (or one standing in for it) whose lengths a test + changed through the core's primitives: the core re-derives its eager + state and every other extension refreshes, whether or not the lengths + changed. + """ + core = metadata.core() + core.resync_length_derived_state() + for extension in metadata.extensions: + if extension is not core: + extension.refresh_from_draft_lengths() + core.clear_lengths_changed() + + +def give_stub_a_draft(stub, block_offsets): + """Give a stub target metadata a draft metadata with ``block_offsets``. + + The draft is a stub too: ``stub.draft`` / ``stub.find("draft_or_none")`` + return it, and its ``lengths_for_kernel_update()`` hands out the stub's + own length buffers (a draft loop on a stub runs on the stub itself). + """ + from types import SimpleNamespace + + draft = SimpleNamespace(kv_cache_block_offsets=block_offsets) + stub.draft = draft + stub.find = lambda name, default=None: draft if name == "draft_or_none" else default + stub.lengths_for_kernel_update = lambda: { + name: getattr(stub, name) + for name in ("seq_lens_cuda", "kv_lens_cuda") + if getattr(stub, name, None) is not None + } + return stub + + def triton_num_compiled(kernel) -> int: """Specializations of a ``triton.jit`` kernel compiled in this process. diff --git a/tests/unittest/_torch/modeling/_deepseek_v41_test_utils.py b/tests/unittest/_torch/modeling/_deepseek_v41_test_utils.py index d9e9eb0ef8b3..6804438612b7 100644 --- a/tests/unittest/_torch/modeling/_deepseek_v41_test_utils.py +++ b/tests/unittest/_torch/modeling/_deepseek_v41_test_utils.py @@ -406,8 +406,7 @@ def run( p + n for p, n in zip(starts[num_contexts:], lengths[num_contexts:]) ] metadata.kv_cache_params.num_cached_tokens_per_seq = list(starts) - metadata.prepare_context_replay(reqs[:num_contexts]) - metadata.prepare() + metadata.prepare(context_requests=reqs[:num_contexts]) positions = torch.cat( [torch.arange(p, p + n, device="cuda", dtype=torch.int32) for p, n in zip(starts, lengths)] ).view(1, -1) diff --git a/tests/unittest/_torch/modeling/test_deepseek_v41_ced.py b/tests/unittest/_torch/modeling/test_deepseek_v41_ced.py index 2514f440ca26..364164e8f421 100644 --- a/tests/unittest/_torch/modeling/test_deepseek_v41_ced.py +++ b/tests/unittest/_torch/modeling/test_deepseek_v41_ced.py @@ -120,18 +120,17 @@ def test_prepared_encoder_replay_preserves_live_speculative_endpoints( req.py_ced_replay = EncoderReplay( 0, id(metadata.kv_cache_manager.kv_cache_map[0]), 256, start ) - metadata.prepare_context_replay([req]) - metadata.prepare() + metadata.prepare(context_requests=[req]) # Host reservation ends at134; accepting one of five drafts moves the # actual six verification rows back to [124,130). The allocator may # only have pages covering that live interval. metadata.kv_lens_cuda[:2].copy_(torch.tensor([end, 130], device="cuda")) - metadata.on_update_kv_lens() + metadata._refresh_live_endpoints() positions = metadata.csa2_positions.clone() swa_slots = {layer: slots.clone() for layer, slots in metadata.csa2_swa_write_slots.items()} for _ in range(2): - metadata.begin_model_forward() - metadata.on_update_kv_lens() + metadata.reset_forward_state() + metadata._refresh_live_endpoints() assert metadata.kv_lens_cuda[:2].tolist() == [end, 130] torch.testing.assert_close(metadata.csa2_positions, positions) assert metadata.csa2_positions[-6:].tolist() == list(range(124, 130)) @@ -147,13 +146,11 @@ def test_prepared_encoder_replay_preserves_live_speculative_endpoints( req.context_current_position = end metadata.seq_lens = torch.tensor([1, 6], dtype=torch.int32) metadata.kv_cache_params.num_cached_tokens_per_seq = [end, 128] - metadata.prepare_context_replay([req]) - metadata.prepare() + metadata.prepare(context_requests=[req]) assert metadata.csa2_replay_mode is None assert metadata.csa2_replay_start_positions.tolist() == [start, 0] req.py_ced_replay = None - metadata.prepare_context_replay([req]) - metadata.prepare() + metadata.prepare(context_requests=[req]) assert metadata.csa2_replay_start_positions.tolist() == [0, 0] diff --git a/tests/unittest/_torch/modeling/test_deepseek_v41_multimodal_backbone.py b/tests/unittest/_torch/modeling/test_deepseek_v41_multimodal_backbone.py index 3ad8fc7aba27..a052f5226686 100644 --- a/tests/unittest/_torch/modeling/test_deepseek_v41_multimodal_backbone.py +++ b/tests/unittest/_torch/modeling/test_deepseek_v41_multimodal_backbone.py @@ -54,7 +54,7 @@ def backbone() -> tuple[v41.DeepseekV41Model, SimpleNamespace, torch.Tensor, tor model.embed_tokens = Mock(return_value=embeddings) seq_lens = torch.tensor([4], dtype=torch.int32) metadata = SimpleNamespace( - begin_model_forward=Mock(), + find_ext=lambda cls: None, num_tokens=4, num_contexts=1, all_rank_num_tokens=[4], diff --git a/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py b/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py index 8527d530d6d3..149e520730e8 100644 --- a/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py +++ b/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py @@ -1238,8 +1238,8 @@ def test_deepseek_v4_sparse_ratios_prefer_checkpoint_defaults(tmp_path, monkeypa assert model_config.sparse_attention_config.compress_ratios == [128, 128, 4, 128, 4, 128, 1, 4] # V4 sparse MLA hardcodes window_size==128 (FMHA kernel TileSizeKV; see - # the runtime assertion in deepseek_v4.py:DeepseekV4TrtllmAttentionMetadata - # __post_init__), so this is the only legal value here. + # the runtime assertion in DeepseekV4Metadata.allocate), so this is the + # only legal value here. assert model_config.sparse_attention_config.window_size == 128 @@ -1559,7 +1559,7 @@ def test_deepseek_v4_sanity( prompt_lens=prompt_lens, max_num_tokens=8192, mapping=mapping, - sparse_attention_config=sparse_attn_config, + sparse_metadata_params=sparse_attn_config.to_sparse_metadata_params(), ) position_ids = [] diff --git a/tests/unittest/_torch/modeling/test_modeling_deepseekv41.py b/tests/unittest/_torch/modeling/test_modeling_deepseekv41.py index 0b29fa0cc4ea..0599c0825309 100644 --- a/tests/unittest/_torch/modeling/test_modeling_deepseekv41.py +++ b/tests/unittest/_torch/modeling/test_modeling_deepseekv41.py @@ -24,11 +24,16 @@ import pytest import torch from _torch.attention.sparse.csa2._utils import _FakeMetadata +from _torch.helpers import bare_extension from torch import nn from transformers import AutoConfig from utils.util import skip_blackwell_geforce, skip_pre_blackwell -from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata +from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import ( + CSA2Metadata, + CSA2TrtllmMetadata, +) from tensorrt_llm._torch.configs import deepseek_v41 as v41_config from tensorrt_llm._torch.configs.deepseek_v41 import ( DeepseekV41Config, @@ -1902,13 +1907,19 @@ def shared_walk(self, weights, skip_modules): class _ReplayMetadata(_FakeMetadata): - _get_csa2_buffer = CSA2TrtllmMetadata._get_csa2_buffer - _copy_host = CSA2TrtllmMetadata._copy_host - _copy_csa2_tensor = CSA2TrtllmMetadata._copy_csa2_tensor - get_adp_token_counts = CSA2TrtllmMetadata.get_adp_token_counts - set_adp_token_counts = CSA2TrtllmMetadata.set_adp_token_counts + _get_csa2_buffer = CSA2Metadata._get_csa2_buffer + _copy_host = CSA2Metadata._copy_host + _copy_csa2_tensor = CSA2Metadata._copy_csa2_tensor decoder_replay_plan = None padded_num_tokens = None + adp_input_token_count = CSA2Metadata.adp_input_token_count + + def get_adp_token_counts(self) -> list[int]: + return [self.adp_input_token_count, *CSA2Metadata.adp_token_counts.fget(self)] + + def set_adp_token_counts(self, counts: list[list[int]]) -> list[int]: + CSA2Metadata.receive_adp_token_counts(self, counts[1:], counts[0]) + return counts[0] def __init__(self, lengths, num_contexts): super().__init__(lengths, num_contexts) @@ -2228,19 +2239,20 @@ def test_final_window_replay_freezes_endpoints_and_preserves_mixed_consumers(ful metadata = _ReplayMetadata([10, 3, 6], num_contexts=2) metadata.kv_cache_params.num_cached_tokens_per_seq = [0, 3, 100] - metadata.kv_lens_cuda = torch.tensor([10, 6, 103]) # Live speculative endpoint is corrected. + # The live speculative endpoint: the target's prepare applied the overlap correction. + metadata.kv_lens_offsets = torch.tensor([0, 0, -3], dtype=torch.int32) reqs = [ SimpleNamespace(prompt_len=50, py_ced_replay=None, py_return_context_logits=full_output), SimpleNamespace(prompt_len=6, py_ced_replay=None), ] - CSA2TrtllmMetadata.prepare_context_replay(metadata, reqs) + CSA2Metadata.prepare_context_replay(metadata, reqs) reqs[0].prompt_len = 10 # Later scheduling must not change this forward's selection. plan = plan_decoder_replay(metadata, 8, reqs, private_decoder=True) assert plan.replay_seq_lens.tolist() == ([10, 3, 6] if full_output else [0, 3, 6]) assert plan.swa_floors == ([0, 0, 0] if full_output else [42, 0, 0]) assert plan.rows.tolist() == list(range(0 if full_output else 10, 19)) - enter_decoder_replay(metadata, plan) - assert metadata.csa2_positions[-6:].tolist() == list(range(97, 103)) + replay = enter_decoder_replay(metadata, plan) + assert replay.csa2_positions[-6:].tolist() == list(range(97, 103)) exit_decoder_replay(metadata, plan) assert metadata.seq_lens.tolist() == [10, 3, 6] assert metadata.kv_cache_params.num_cached_tokens_per_seq == [0, 3, 100] @@ -2263,18 +2275,23 @@ def test_final_window_zero_queries_still_publish_global_and_restore_outputs(peer plan.saved_all_rank_num_tokens = [129, 257] plan.replay_all_rank_num_tokens = [0, peer_tokens] positions = torch.arange(129) - _, _, replay_state = model._enter_bounded_replay(plan, metadata, positions, positions, state) + _, _, replay_state, replay = model._enter_bounded_replay( + plan, metadata, positions, positions, state + ) + assert replay is metadata.decoder_replay_metadata assert replay_state.residual.shape == (0, 4, 64) layer.self_attn.prepare_global_cache.assert_called_once() assert layer.self_attn.prepare_global_cache.call_args.args[0].shape[0] == 129 - assert metadata.csa2_precomputed_kv_layers == {20} - assert metadata.prepare_calls == 0 - assert metadata.all_rank_num_tokens == [0, peer_tokens] + assert replay.csa2_precomputed_kv_layers == {20} + assert metadata.prepare_calls == replay.prepare_calls == 0 + assert replay.all_rank_num_tokens == [0, peer_tokens] + assert replay.num_tokens == 0 restored = model._exit_bounded_replay(plan, metadata, replay_state.residual[:, 0]) assert restored.shape == (129, 64) assert not restored.count_nonzero() assert metadata.csa2_decoder_capture_lens == (0,) - assert metadata.all_rank_num_tokens == [129, 257] + # The target keeps its own token counts; only the replay metadata had the exchanged ones. + assert metadata.all_rank_num_tokens is None assert metadata.seq_lens.tolist() == [129] assert metadata.kv_cache_params.num_cached_tokens_per_seq == [0] @@ -2286,7 +2303,7 @@ def test_final_window_prefill_graph_capability_keeps_intermediate_history(): metadata = _ReplayMetadata([257], num_contexts=1) reqs = [SimpleNamespace(prompt_len=2048, py_ced_replay=None)] - CSA2TrtllmMetadata.prepare_context_replay(metadata, reqs, allow_final_window=False) + CSA2Metadata.prepare_context_replay(metadata, reqs, allow_final_window=False) plan = plan_decoder_replay(metadata, 128, reqs, private_decoder=True) assert plan.replay_seq_lens.tolist() == [128] assert plan.rows.tolist() == list(range(129, 257)) @@ -2403,11 +2420,12 @@ def test_model_boundary_preserves_full_cache_and_compacts_query_rows( assert plan.rows.tolist() == list(range(1, sum(lengths))) positions = torch.arange(sum(lengths)) ids = positions + 100 - new_positions, new_ids, new_state = model._enter_bounded_replay( + new_positions, new_ids, new_state, replay = model._enter_bounded_replay( plan, metadata, positions, ids, state ) - assert metadata.csa2_precomputed_kv_layers == {20} - assert metadata.csa2_replay_query_rows is plan.rows + assert replay is metadata.decoder_replay_metadata + assert replay.csa2_precomputed_kv_layers == {20} + assert replay.csa2_replay_query_rows is plan.rows writer = layer.self_attn.prepare_global_cache writer.assert_called_once() actual, passed_metadata = writer.call_args.args @@ -2421,8 +2439,11 @@ def test_model_boundary_preserves_full_cache_and_compacts_query_rows( torch.testing.assert_close(new_ids, ids[plan.rows]) torch.testing.assert_close(new_state.residual, state.residual[plan.rows]) torch.testing.assert_close(new_state.pre_mix, state.pre_mix[plan.rows]) - assert metadata.seq_lens.tolist() == [128, 7, 6] - assert metadata.kv_cache_params.num_cached_tokens_per_seq == [1, 0, 0] + assert replay.seq_lens.tolist() == [128, 7, 6] + assert replay.kv_cache_params.num_cached_tokens_per_seq == [1, 0, 0] + # The target keeps the encoder pass's batch. + assert metadata.seq_lens.tolist() == lengths + assert metadata.kv_cache_params.num_cached_tokens_per_seq == [0, 0, 0] # A later request must not see the cache-only marker from this forward. restored = model._exit_bounded_replay(plan, metadata, new_state.residual) assert restored.shape == state.residual.shape @@ -2477,7 +2498,7 @@ def test_remote_tail_source_reuses_boundary_collapse( metadata = _ReplayMetadata(lengths, num_contexts=num_contexts) metadata.mapping = model.model_config.mapping metadata.csa2_remote_tail_mode = None if context_only else "source" - metadata.begin_model_forward = Mock() + metadata.reset_forward_state = Mock() post_mapping = Mock(return_value=resolved.residual) state = resolved if deferred: @@ -2515,7 +2536,7 @@ def test_remote_tail_source_reuses_boundary_collapse( assert post_args["post_layer_mix"] is state.post_mix assert post_args["comb_res_mix"] is state.comb_mix encoder.assert_called_once() - metadata.begin_model_forward.assert_called_once() + metadata.reset_forward_state.assert_called_once() writer = layer.self_attn.prepare_global_cache if ced_precompute: writer.assert_called_once() @@ -2608,7 +2629,7 @@ def test_remote_tail_attention_dp_generation_keeps_all_layers( metadata.is_cuda_graph = is_cuda_graph metadata.csa2_remote_tail_mode = remote_mode metadata.csa2_remote_tail_starts = [1024] if remote_mode == "destination" else [] - metadata.begin_model_forward = Mock() + metadata.reset_forward_state = Mock() enter_decoder = Mock() monkeypatch.setattr(v41, "enter_remote_tail_decoder", enter_decoder) @@ -2625,7 +2646,7 @@ def test_remote_tail_attention_dp_generation_keeps_all_layers( assert enter_decoder.call_count == int(remote_mode == "destination") assert boundary.self_attn.prepare_global_cache.call_count == int(remote_mode == "destination") assert metadata.csa2_replay_query_rows is None - metadata.begin_model_forward.assert_called_once() + metadata.reset_forward_state.assert_called_once() def test_context_decoder_skipping_requires_bounded_replay( @@ -2687,8 +2708,9 @@ def test_context_only_forward_routes_warmup_to_source_and_refuses_destination(mo nn.Module.__init__(model) model.disagg_context_only = True metadata = SimpleNamespace( - csa2_remote_tail_mode=mode, begin_model_forward=Mock(), all_rank_num_tokens=None + csa2_remote_tail_mode=mode, reset_forward_state=Mock(), all_rank_num_tokens=None ) + metadata.find_ext = lambda cls: metadata result = object() def source_forward(self, passed_metadata): @@ -2703,7 +2725,7 @@ def source_forward(self, passed_metadata): model.forward(metadata) else: assert model.forward(metadata) is result - metadata.begin_model_forward.assert_called_once() + metadata.reset_forward_state.assert_called_once() def test_context_only_model_rejects_speculative_decoding_before_construction(monkeypatch): @@ -3009,6 +3031,8 @@ def test_model_boundary_resets_routing_and_preserves_layer_guard(monkeypatch): from tensorrt_llm._torch.models.modeling_deepseekv41 import DeepseekV41Model metadata = CSA2TrtllmMetadata.__new__(CSA2TrtllmMetadata) + metadata.replace_ext(bare_extension(CSA2Metadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata.reset_routing() layout = CSA2Layout((1, 1), (0,), (0,)) model = DeepseekV41Model.__new__(DeepseekV41Model) diff --git a/tests/unittest/_torch/modeling/test_modeling_gemma3.py b/tests/unittest/_torch/modeling/test_modeling_gemma3.py index 879d21a33917..1159b6159cc3 100644 --- a/tests/unittest/_torch/modeling/test_modeling_gemma3.py +++ b/tests/unittest/_torch/modeling/test_modeling_gemma3.py @@ -228,12 +228,15 @@ def _verify_params_flushed_upon_prepare(self, if isinstance(attn_metadata, FlashInferAttentionMetadata): # Right after forward call with custom mask, plan_params will have non-trivial attention_mask_data. # One for global-prefill, other for local-prefill. - self.assertEqual(len(attn_metadata._plan_params_to_wrappers), 2) - for plan_params in attn_metadata._plan_params_to_wrappers.keys(): + self.assertEqual(len(attn_metadata.core()._plan_params_to_wrappers), + 2) + for plan_params in attn_metadata.core( + )._plan_params_to_wrappers.keys(): assert plan_params.attention_mask_data is not None # Prepare should flush the params with non-trivial attention_mask_data. attn_metadata.prepare() - self.assertEqual(len(attn_metadata._plan_params_to_wrappers), 0) + self.assertEqual(len(attn_metadata.core()._plan_params_to_wrappers), + 0) # Allow room for small fraction of elements to fail. This is to mitigate flakiness. def _assert_most_elems_close(self, actual_value, ref_value, atol, rtol, diff --git a/tests/unittest/_torch/modeling/test_modeling_gemma4.py b/tests/unittest/_torch/modeling/test_modeling_gemma4.py index c2fe89d51232..08d24d11bf4e 100644 --- a/tests/unittest/_torch/modeling/test_modeling_gemma4.py +++ b/tests/unittest/_torch/modeling/test_modeling_gemma4.py @@ -2002,15 +2002,17 @@ def test_vswa_per_pool_page_indices(self): ) # VSWA detection must populate layer-to-pool mapping - self.assertIsNotNone(metadata._vswa_layer_to_pool, "VSWA layer_to_pool mapping not created") + self.assertIsNotNone( + metadata.core()._vswa_layer_to_pool, "VSWA layer_to_pool mapping not created" + ) # Sliding and full layers must map to different pools sliding_pools = { - metadata._vswa_layer_to_pool[i] + metadata.core()._vswa_layer_to_pool[i] for i in range(config.num_hidden_layers) if config.layer_types[i] == "sliding_attention" } full_pools = { - metadata._vswa_layer_to_pool[i] + metadata.core()._vswa_layer_to_pool[i] for i in range(config.num_hidden_layers) if config.layer_types[i] == "full_attention" } @@ -2023,11 +2025,13 @@ def test_vswa_per_pool_page_indices(self): metadata.prepare() self.assertIsNotNone( - metadata._vswa_pool_indices_cache, + metadata.core()._vswa_pool_indices_cache, "Per-pool indices cache not populated after prepare()", ) self.assertEqual( - len(metadata._vswa_pool_indices_cache), 2, "Expected 2 pool entries (sliding + full)" + len(metadata.core()._vswa_pool_indices_cache), + 2, + "Expected 2 pool entries (sliding + full)", ) # Each pool's indices are fetched via different layer_id → pool_id @@ -2160,7 +2164,7 @@ def test_vswa_swap_restores_correct_pool(self): # Swap to sliding → check buffer matches sliding pool metadata.swap_paged_kv_indices_for_layer(sliding_layer) - actual = metadata._paged_kv_indices[:total_blocks].cpu() + actual = metadata.core()._paged_kv_indices[:total_blocks].cpu() self.assertTrue( torch.equal(actual, sliding_expected), "After swap to sliding layer, buffer should match sliding pool", @@ -2168,7 +2172,7 @@ def test_vswa_swap_restores_correct_pool(self): # Swap to full → check buffer matches full pool metadata.swap_paged_kv_indices_for_layer(full_layer) - actual = metadata._paged_kv_indices[:total_blocks].cpu() + actual = metadata.core()._paged_kv_indices[:total_blocks].cpu() self.assertTrue( torch.equal(actual, full_expected), "After swap to full layer, buffer should match full pool", @@ -2176,7 +2180,7 @@ def test_vswa_swap_restores_correct_pool(self): # Swap back to sliding → check buffer is restored metadata.swap_paged_kv_indices_for_layer(sliding_layer) - actual = metadata._paged_kv_indices[:total_blocks].cpu() + actual = metadata.core()._paged_kv_indices[:total_blocks].cpu() self.assertTrue( torch.equal(actual, sliding_expected), "After swap back to sliding, buffer should be restored", @@ -2347,17 +2351,17 @@ def test_vswa_pool_cache_not_aliased(self): with torch.inference_mode(): metadata.prepare() - if metadata._vswa_layer_to_pool is None: + if metadata.core()._vswa_layer_to_pool is None: kv_cache_manager.shutdown() self.skipTest("No VSWA detected") - primary_pool_id = metadata._vswa_layer_to_pool.get(0, 0) + primary_pool_id = metadata.core()._vswa_layer_to_pool.get(0, 0) # The primary pool cache buffer must NOT be the same object as # _paged_kv_indices — that's the aliasing bug. self.assertIsNot( - metadata._vswa_pool_indices_cache[primary_pool_id], - metadata._paged_kv_indices, + metadata.core()._vswa_pool_indices_cache[primary_pool_id], + metadata.core()._paged_kv_indices, "Primary pool cache is aliased to _paged_kv_indices! " "Pool swaps will corrupt the primary pool's cached indices.", ) @@ -2365,7 +2369,7 @@ def test_vswa_pool_cache_not_aliased(self): # Record the original primary pool indices total_blocks = metadata.num_generation_blocks + metadata.num_context_blocks pool0_original = ( - metadata._vswa_pool_indices_cache[primary_pool_id][:total_blocks].cpu().clone() + metadata.core()._vswa_pool_indices_cache[primary_pool_id][:total_blocks].cpu().clone() ) # Find a full-attention (secondary pool) layer @@ -2385,7 +2389,7 @@ def test_vswa_pool_cache_not_aliased(self): metadata.swap_paged_kv_indices_for_layer(sliding_layer) # After round-trip swap, primary pool cache must still have original values - pool0_after = metadata._vswa_pool_indices_cache[primary_pool_id][:total_blocks].cpu() + pool0_after = metadata.core()._vswa_pool_indices_cache[primary_pool_id][:total_blocks].cpu() self.assertTrue( torch.equal(pool0_original, pool0_after), f"Primary pool indices corrupted after pool swap round-trip: " @@ -3520,7 +3524,7 @@ def _expected_decode_block_table( width: int, ) -> torch.Tensor: """Build the expected compact table from one VSWA pool's host indices.""" - pool_indices = metadata._host_pool_indices[pool_id].numpy() + pool_indices = metadata.core()._host_pool_indices[pool_id].numpy() expected = torch.zeros((rows, width), dtype=torch.int32) source_offset = metadata.num_context_blocks for row, page_count in enumerate(page_counts): @@ -3550,14 +3554,14 @@ def test_shared_kv_draft_view(self) -> None: accepted_tokens, num_contexts=0, ) - expected_kv_lens = metadata._cached_token_lens[:2] + accepted_tokens + expected_kv_lens = metadata.core()._cached_token_lens[:2] + accepted_tokens for layer, query in zip(layers, queries, strict=True): layer.forward(query, None, None, draft_metadata) self.assertIs(draft_metadata.kv_cache_manager, metadata.kv_cache_manager) torch.testing.assert_close( - draft_metadata._draft_kv_runtime_lens[:2], + draft_metadata.core()._draft_kv_runtime_lens[:2], expected_kv_lens, atol=0, rtol=0, @@ -3586,7 +3590,7 @@ def test_cuda_graph_trtllm_gen_block_table_transitions(self) -> None: ) torch.cuda.synchronize() - for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + for plan_params, wrappers in metadata.core()._plan_params_to_wrappers.items(): with self.subTest(head_dim=plan_params.head_dim): block_tables = wrappers.decode_wrapper._block_tables expected = self._expected_decode_block_table( @@ -3617,7 +3621,7 @@ def test_cuda_graph_trtllm_gen_host_table_growth_keeps_device_pointer(self) -> N ) initial_state = {} - for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + for plan_params, wrappers in metadata.core()._plan_params_to_wrappers.items(): self.assertIsNotNone(plan_params.kv_pool_id) block_tables = wrappers.decode_wrapper._block_tables self.assertGreaterEqual(block_tables.size(1), 65) @@ -3632,7 +3636,7 @@ def test_cuda_graph_trtllm_gen_host_table_growth_keeps_device_pointer(self) -> N self._prepare_decode_page_counts(metadata, [0, 1], new_page_counts) torch.cuda.synchronize() - for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + for plan_params, wrappers in metadata.core()._plan_params_to_wrappers.items(): with self.subTest(head_dim=plan_params.head_dim): block_tables = wrappers.decode_wrapper._block_tables old_device_ptr, old_host_ptr = initial_state[plan_params.kv_pool_id] @@ -4378,7 +4382,7 @@ def _run_cuda_graph_real_headdim( if expect_split_kv: split_kv_head_dims = set() - for plan_params, wrappers in cg_metadata._plan_params_to_wrappers.items(): + for plan_params, wrappers in cg_metadata.core()._plan_params_to_wrappers.items(): decode_wrapper = wrappers.decode_wrapper if decode_wrapper is None or decode_wrapper._backend != "fa2": continue diff --git a/tests/unittest/_torch/modeling/test_modeling_llama.py b/tests/unittest/_torch/modeling/test_modeling_llama.py index a7b79b783d2b..05ed64b1f2ab 100644 --- a/tests/unittest/_torch/modeling/test_modeling_llama.py +++ b/tests/unittest/_torch/modeling/test_modeling_llama.py @@ -15,6 +15,8 @@ from utils.util import default_dtype, getSMVersion import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features.spec_decoding import \ + SpecDecodingConfig from tensorrt_llm._torch.attention.backends.utils import get_attention_backend from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import ModelConfig @@ -546,24 +548,24 @@ def run_forward(input_ids, position_ids, attn_metadata): kv_cache_manager=kv_cache_manager, request_ids=request_ids, prompt_lens=prompt_lens, - is_spec_decoding_enabled=is_spec_decoding_enabled, - use_spec_decoding=use_spec_decoding, - is_spec_dec_tree=is_spec_dec_tree, - is_spec_dec_dynamic_tree=is_spec_dec_dynamic_tree, max_num_heads_per_kv=num_heads_per_kv, + spec_decoding=SpecDecodingConfig( + attention_needs_spec_dec_mode=is_spec_decoding_enabled, + is_spec_dec_tree=is_spec_dec_tree, + is_spec_dec_dynamic_tree=is_spec_dec_dynamic_tree, + max_draft_len=max_total_draft_tokens, + max_total_draft_tokens=max_total_draft_tokens, + tree_buffer_dim=spec_tree_mgr._internal_buf_dim, + ), ) cache = kv_cache_manager.kv_cache_map[request.py_request_id] assert cache.resize(input_ids.size(-1) + gen_input_ids_0.size(-1)) attn_metadata_gen_phase_0.prepare() attn_metadata_gen_phase_0.update_spec_dec_param( batch_size=batch_size, - is_spec_decoding_enabled=is_spec_decoding_enabled, - is_spec_dec_dynamic_tree=is_spec_dec_dynamic_tree, - is_spec_dec_tree=is_spec_dec_tree, - max_draft_len=max_total_draft_tokens, - max_total_draft_tokens=max_total_draft_tokens, spec_tree_manager=spec_tree_mgr, ) + attn_metadata_gen_phase_0.use_spec_decoding = use_spec_decoding gen_position_ids_0 = [ torch.full((gen_input_ids_0.size(-1), ), @@ -601,16 +603,8 @@ def run_forward(input_ids, position_ids, attn_metadata): num_cached_tokens_per_seq_1 = [ input_ids.size(-1) + request.py_num_accepted_draft_tokens + 1 ] - attn_metadata_gen_phase_0.seq_lens = torch.tensor( - [gen_input_ids_1.size(-1)], dtype=torch.int) - attn_metadata_gen_phase_0.kv_cache_params.num_cached_tokens_per_seq = num_cached_tokens_per_seq_1 - - attn_metadata_gen_phase_0.spec_decoding_position_offsets = None - attn_metadata_gen_phase_0.spec_decoding_packed_mask = None - attn_metadata_gen_phase_0.spec_decoding_generation_lengths = None assert cache.resize(num_cached_tokens_per_seq_1[0] + gen_input_ids_1.size(-1)) - attn_metadata_gen_phase_0.prepare() is_tree_phase1 = is_spec_dec_tree if get_sm_version() < 100 else False spec_tree_mgr_phase1 = None spec_metadata_phase1 = None @@ -628,13 +622,34 @@ def run_forward(input_ids, position_ids, attn_metadata): max_total_draft_tokens=max_draft_1, spec_dec_mode=SpeculativeDecodingMode.EAGLE3_ONE_MODEL, ) - attn_metadata_gen_phase_0.update_spec_dec_param( + # The second step's draft settings differ, and spec-dec settings are + # creation configuration: it gets a metadata of its own. + attn_metadata_gen_phase_1 = metadata_cls( + seq_lens=torch.tensor([gen_input_ids_1.size(-1)], dtype=torch.int), + num_contexts=0, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq_1, + ), + max_num_requests=1, + max_num_tokens=8192, + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + prompt_lens=prompt_lens, + max_num_heads_per_kv=num_heads_per_kv, + spec_decoding=SpecDecodingConfig( + attention_needs_spec_dec_mode=is_spec_decoding_enabled, + is_spec_dec_tree=is_tree_phase1, + is_spec_dec_dynamic_tree=is_tree_phase1, + max_draft_len=gen_input_ids_1.size(-1) - 1, + max_total_draft_tokens=gen_input_ids_1.size(-1) - 1, + tree_buffer_dim=(spec_tree_mgr_phase1._internal_buf_dim + if spec_tree_mgr_phase1 is not None else 0), + ), + ) + attn_metadata_gen_phase_1.prepare() + attn_metadata_gen_phase_1.update_spec_dec_param( batch_size=batch_size, - is_spec_decoding_enabled=is_spec_decoding_enabled, - is_spec_dec_tree=is_tree_phase1, - is_spec_dec_dynamic_tree=is_tree_phase1, - max_draft_len=gen_input_ids_1.size(-1) - 1, - max_total_draft_tokens=gen_input_ids_1.size(-1) - 1, spec_metadata=spec_metadata_phase1, spec_tree_manager=spec_tree_mgr_phase1) @@ -649,7 +664,7 @@ def run_forward(input_ids, position_ids, attn_metadata): with torch.inference_mode(): gen_logits_1 = run_forward(input_ids=gen_input_ids_1, position_ids=gen_position_ids_1, - attn_metadata=attn_metadata_gen_phase_0) + attn_metadata=attn_metadata_gen_phase_1) torch.cuda.synchronize() @@ -659,30 +674,6 @@ def run_forward(input_ids, position_ids, attn_metadata): device=device) num_cached_tokens_per_seq_ref = [input_ids.size(-1)] - attn_metadata_ref = metadata_cls( - seq_lens=torch.tensor([gen_input_ids_ref.size(-1)], - dtype=torch.int), - num_contexts=0, - kv_cache_params=KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=num_cached_tokens_per_seq_ref, - ), - max_num_requests=1, - max_num_tokens=8192, - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - prompt_lens=prompt_lens, - is_spec_decoding_enabled=is_spec_decoding_enabled, - use_spec_decoding=use_spec_decoding, - is_spec_dec_tree=is_spec_dec_tree, - is_spec_dec_dynamic_tree=False, - max_num_heads_per_kv=num_heads_per_kv, - ) - - attn_metadata_ref.spec_decoding_position_offsets = None - attn_metadata_ref.spec_decoding_packed_mask = None - attn_metadata_ref.spec_decoding_generation_lengths = None - attn_metadata_ref.prepare() is_tree_ref = is_spec_dec_tree if get_sm_version() < 100 else False spec_tree_mgr_ref = None spec_metadata_ref = None @@ -700,13 +691,33 @@ def run_forward(input_ids, position_ids, attn_metadata): max_total_draft_tokens=max_draft_ref, spec_dec_mode=SpeculativeDecodingMode.EAGLE3_ONE_MODEL, ) + attn_metadata_ref = metadata_cls( + seq_lens=torch.tensor([gen_input_ids_ref.size(-1)], + dtype=torch.int), + num_contexts=0, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq_ref, + ), + max_num_requests=1, + max_num_tokens=8192, + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + prompt_lens=prompt_lens, + max_num_heads_per_kv=num_heads_per_kv, + spec_decoding=SpecDecodingConfig( + attention_needs_spec_dec_mode=is_spec_decoding_enabled, + is_spec_dec_tree=is_tree_ref, + is_spec_dec_dynamic_tree=is_tree_ref, + max_draft_len=gen_input_ids_ref.size(-1) - 1, + max_total_draft_tokens=gen_input_ids_ref.size(-1) - 1, + tree_buffer_dim=(spec_tree_mgr_ref._internal_buf_dim + if spec_tree_mgr_ref is not None else 0), + ), + ) + attn_metadata_ref.prepare() attn_metadata_ref.update_spec_dec_param( batch_size=batch_size, - is_spec_decoding_enabled=is_spec_decoding_enabled, - is_spec_dec_tree=is_tree_ref, - is_spec_dec_dynamic_tree=is_tree_ref, - max_draft_len=gen_input_ids_ref.size(-1) - 1, - max_total_draft_tokens=gen_input_ids_ref.size(-1) - 1, spec_metadata=spec_metadata_ref, spec_tree_manager=spec_tree_mgr_ref) diff --git a/tests/unittest/_torch/modeling/test_qsa_runtime_wiring.py b/tests/unittest/_torch/modeling/test_qsa_runtime_wiring.py index 3f20cbdaff18..b4d76f9fdc82 100644 --- a/tests/unittest/_torch/modeling/test_qsa_runtime_wiring.py +++ b/tests/unittest/_torch/modeling/test_qsa_runtime_wiring.py @@ -5,16 +5,18 @@ import pytest import torch +from _torch.helpers import bare_extension, construct_extension, stub_extension from tensorrt_llm._torch.attention.backends.interface import PredefinedAttentionMask from tensorrt_llm._torch.attention.backends.sparse.hooks import get_sparse_attention_hooks +from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension from tensorrt_llm._torch.attention.backends.sparse.qsa import ( QSAAttentionMetadata, QSASparseMetadataParams, QSASparseParams, ) +from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAMetadata from tensorrt_llm._torch.attention.backends.sparse.qsa.module import QSASparseHooks -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import Qwen4ExpHybridCacheManager from tensorrt_llm._torch.pyexecutor._util import _create_kv_cache_manager, get_kv_cache_manager_cls @@ -224,18 +226,19 @@ def test_qsa_empty_batch_keeps_the_regular_backend_path() -> None: assert output is None -def test_qsa_metadata_allows_a_pp_rank_without_local_sparse_layers(monkeypatch) -> None: - monkeypatch.setattr(TrtllmAttentionMetadata, "__post_init__", lambda self: None) +def test_qsa_metadata_allows_a_pp_rank_without_local_sparse_layers() -> None: manager = object.__new__(Qwen4ExpHybridCacheManager) manager.qsa_position_layer_id = None metadata = object.__new__(QSAAttentionMetadata) metadata.kv_cache_manager = manager - metadata.sparse_metadata_params = QSASparseMetadataParams( + metadata.replace_ext(bare_extension(MTPIndexShareExtension, metadata)) + qsa = stub_extension(QSAMetadata, metadata) + qsa.sparse_metadata_params = QSASparseMetadataParams( token_topk=2048, compress_ratio=4, ) - - metadata.__post_init__() + metadata.replace_ext(qsa) + construct_extension(qsa) assert not metadata.qsa_has_local_layers diff --git a/tests/unittest/_torch/modeling/test_qwen4_exp_support.py b/tests/unittest/_torch/modeling/test_qwen4_exp_support.py index 79fb2133d280..03e4b56cdac0 100644 --- a/tests/unittest/_torch/modeling/test_qwen4_exp_support.py +++ b/tests/unittest/_torch/modeling/test_qwen4_exp_support.py @@ -1529,6 +1529,7 @@ def test_ple_prefill_reuses_prepared_device_metadata(monkeypatch) -> None: model.ple_layer_index = 0 model.layers = [SimpleNamespace(ple=SimpleNamespace())] model.eos_token_id = 2 + model._resolved_ple_pools = {} captured = {} @@ -1585,6 +1586,8 @@ def fake_build( assert captured["state_indices"].dtype == torch.long assert captured["kwargs"]["host_seq_lens"] == [2, 3] assert captured["input_ids"].shape == (5,) + # The model keeps the pools it resolved; the metadata takes no writes. + assert model._resolved_ple_pools == {0: (conv_state, ngram_context)} torch.testing.assert_close(conv_state.flatten(), torch.tensor([1.0, 0.0], device=device)) torch.testing.assert_close( ngram_context.flatten(), torch.tensor([11, 2], dtype=torch.long, device=device) diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py index 1bd3425c6853..d0aff5b23a33 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py @@ -9,6 +9,7 @@ """ import copy +from contextlib import contextmanager from types import SimpleNamespace import pytest @@ -19,6 +20,7 @@ Glm5NextMtpStep, _check_speculation, ) +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManager from tensorrt_llm._torch.speculative.utils import get_spec_worker from tensorrt_llm.llmapi.llm_args import AdvancedSamplingMode, MTPDecodingConfig @@ -47,7 +49,10 @@ def update_mamba_states(self, attn_metadata, num_accepted_tokens, state_indices) class _Metadata: - """The attention-metadata surface the draft chain reads and rewrites.""" + """The attention-metadata surface the draft chain reads and rewrites. + + It is its own draft metadata: a draft loop on it runs on it, and refreshing the draft does nothing. + """ def __init__(self, ctx_lens, num_gens, draft_len, kv_lens, device): seq_lens = list(ctx_lens) + [draft_len + 1] * num_gens @@ -69,8 +74,6 @@ def __init__(self, ctx_lens, num_gens, draft_len, kv_lens, device): self.spec_decoding_position_offsets = None self.spec_decoding_generation_lengths = None self.use_spec_decoding = True - self.enable_flash_mla = False - self._saved_tensors = {} self.on_update() @property @@ -84,26 +87,38 @@ def num_ctx_tokens(self): def on_update(self): self._num_ctx_tokens = int(self._seq_lens[: self.num_contexts].sum()) - def on_update_kv_lens(self): - pass + def set_seq_lens(self, rows, *, host=None, device=None, update_counts=False): + if host is not None: + self._seq_lens[rows] = host + if device is not None: + self._seq_lens_cuda[rows] = device + if update_counts: + self.on_update() + + def as_generation_rows(self): + self.host_request_types[: self.num_contexts].fill_(1) + self.num_contexts = 0 + self.on_update() - def update_for_spec_dec(self): - pass + def adjust_kv_lens(self, rows, delta): + self.kv_lens_cuda[rows] += delta - def prepare_for_spec_dec(self, *fields): - assert not self._saved_tensors - for f in fields: - self._saved_tensors[f] = getattr(self, f) - setattr(self, f, getattr(self, f).clone()) + def lengths_for_kernel_update(self): + return {"seq_lens_cuda": self._seq_lens_cuda, "kv_lens_cuda": self.kv_lens_cuda} - def restore_from_spec_dec(self): - for f, v in self._saved_tensors.items(): - setattr(self, f, v) - self._saved_tensors.clear() + @contextmanager + def draft_loop(self): + yield self - @property - def has_spec_dec_saved_state(self): - return bool(self._saved_tensors) + def refresh_draft(self): + pass + + def find_ext(self, cls): + # Serves the speculative-decoding flags itself; has no other extension. + return self if cls is SpecDecodingMetadata else None + + def find(self, name, default=None): + return default def snapshot(self): return dict( @@ -265,7 +280,6 @@ def test_step_matches_stock_worker(ctx_lens, accept_depths, draft_len, greedy): stock_in["draft_model"].mtp_layers[0].calls == fused_in["draft_model"].mtp_layers[0].calls ) assert stock_in["attn_metadata"].snapshot() == fused_in["attn_metadata"].snapshot() - assert not fused_in["attn_metadata"].has_spec_dec_saved_state # The same accepted counts committed to the KDA state. s_commits = stock_in["attn_metadata"].kv_cache_manager.commits f_commits = fused_in["attn_metadata"].kv_cache_manager.commits diff --git a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/_draft_step_op_matrix.py b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/_draft_step_op_matrix.py index 93503df0b8d7..7edfaa00a5db 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/_draft_step_op_matrix.py +++ b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/_draft_step_op_matrix.py @@ -493,10 +493,17 @@ def check_tp8_worker_feedback_eager_and_graph() -> None: meta.set_in_mtp_draft_loop = lambda value, m=meta: setattr(m, "in_mtp_draft_loop", value) meta.set_skip_topk = lambda value, m=meta: setattr(m, "indexer_skip_topk", value) meta.set_mtp_num_accepted = lambda value, m=meta: setattr(m, "mtp_num_accepted", value) - meta.on_update = lambda: None - meta.update_for_spec_dec = lambda m=meta, s=seq, k=kv: m.updates.append( - (s.clone(), k.clone()) + meta.reset_mtp_index_share = lambda m=meta: ( + m.set_skip_topk(False), + m.set_in_mtp_draft_loop(False), + m.set_mtp_num_accepted(None), ) + # The stub is its own MTP index-share extension. + meta.find_ext = meta.ext = lambda extension_type, m=meta: m + meta.set_seq_lens = lambda rows, *, host=None, device=None, update_counts=False, m=meta: ( + m._seq_lens.__setitem__(rows, host) if host is not None else None + ) + meta.before_forward = lambda m=meta, s=seq, k=kv: m.updates.append((s.clone(), k.clone())) helper = object.__new__(SmallBatchDraftKernels) helper.use_table, helper.hidden_size = True, HIDDEN helper.DRAFT_LEN = draft_len diff --git a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/test_modeling_v2_small_batch_mtp.py b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/test_modeling_v2_small_batch_mtp.py index 7df17e5014b8..237b8ff720f8 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/test_modeling_v2_small_batch_mtp.py +++ b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/spec/test_modeling_v2_small_batch_mtp.py @@ -74,6 +74,8 @@ def table_reference(logits, *args): class _Metadata: + """A draft metadata stub that is its own MTP index-share extension.""" + def __init__(self, batch, k=5): self._seq_lens = torch.full((batch,), k + 1, dtype=torch.int32) self.seq_lens_cuda = self._seq_lens.clone() @@ -96,10 +98,28 @@ def set_skip_topk(self, value): def set_mtp_num_accepted(self, value): self.mtp_num_accepted = value - def on_update(self): - self.num_tokens = int(self._seq_lens.sum()) + def reset_mtp_index_share(self): + self.set_skip_topk(False) + self.set_in_mtp_draft_loop(False) + self.set_mtp_num_accepted(None) + + def find_ext(self, extension_type): + return self + + ext = find_ext + + def set_seq_lens(self, rows, *, host=None, device=None, update_counts=False): + if host is not None: + self._seq_lens[rows] = host + if device is not None: + self.seq_lens_cuda[rows] = device + if update_counts: + self.num_tokens = int(self._seq_lens.sum()) + + def lengths_for_kernel_update(self): + return {"seq_lens_cuda": self.seq_lens_cuda, "kv_lens_cuda": self.kv_lens_cuda} - def update_for_spec_dec(self): + def before_forward(self): self.updates.append((self.seq_lens_cuda.clone(), self.kv_lens_cuda.clone())) @@ -192,7 +212,10 @@ def run_forward(draft_model, step_inputs, spec_metadata, step): return value, None worker = SimpleNamespace( - _drafted_rows=None, draft_kv_cache_context=draft_context, _run_draft_forward=run_forward + _drafted_rows=None, + draft_kv_cache_context=draft_context, + _run_draft_forward=run_forward, + refresh_draft=metadata.before_forward, ) return worker, helper, inputs, metadata, spec, counts, calls, preflights @@ -542,7 +565,6 @@ def operand(shape, dtype): rows = batch * (runtime_k + 1) meta = _Metadata(batch, runtime_k) - monkeypatch.setattr(worker_module, "MTPIndexShareMetadata", _Metadata) layer, lm_head = object(), object() helper = SimpleNamespace( layer=layer, diff --git a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/_kimi_k3_decode_step.py b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/_kimi_k3_decode_step.py index 916c3ba6d4f4..edae05ca502d 100644 --- a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/_kimi_k3_decode_step.py +++ b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/_kimi_k3_decode_step.py @@ -11,6 +11,7 @@ from importlib import import_module import torch +from _torch.helpers import give_stub_a_draft from tensorrt_llm._device_copy import DeviceCopyBuffer @@ -242,10 +243,10 @@ def accept_kernel(*args): num_seqs=3, num_contexts=0, kv_cache_manager=manager, - draft_kv_cache_block_offsets=torch.zeros(3, 1), mamba_metadata=types.SimpleNamespace(state_indices=torch.arange(3)), kv_lens_cuda=torch.ones(3, dtype=torch.int32), ) + give_stub_a_draft(metadata, torch.zeros(3, 1)) spec = types.SimpleNamespace( runtime_draft_len=7, is_cuda_graph=False, draft_tokens=torch.zeros(3, 7) ) @@ -254,7 +255,7 @@ def accept_kernel(*args): model=types.SimpleNamespace(embed_tokens=torch.nn.Embedding(8, 4)) ) ) - worker._k3_accept(torch.zeros(3, 8), metadata, spec, draft) + worker._k3_accept(torch.zeros(3, 8), metadata, spec, draft, metadata.kv_lens_cuda) assert received[0] is worker._batch_to_slot_buffer.tensor torch.testing.assert_close(received[0], torch.tensor(expected), rtol=0, atol=0) @@ -270,9 +271,17 @@ def check_drafter_inputs_use_current_slot_buffer(monkeypatch, spec_worker, num_c metadata = types.SimpleNamespace( num_contexts=num_contexts, num_seqs=3, - _seq_lens_cuda=torch.ones(3, dtype=torch.int32), - _seq_lens=torch.ones(3, dtype=torch.int32), + seq_lens_cuda=torch.ones(3, dtype=torch.int32), + seq_lens=torch.ones(3, dtype=torch.int32), ) + + def set_seq_lens(rows, *, host=None, device=None, update_counts=False): + if host is not None: + metadata.seq_lens[rows] = host + if device is not None: + metadata.seq_lens_cuda[rows] = device + + metadata.set_seq_lens = set_seq_lens spec = types.SimpleNamespace( hidden_size=4, runtime_draft_len=7, get_hidden_states=lambda count: None ) diff --git a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/test_k3_decode_worker.py b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/test_k3_decode_worker.py index 55e00273903f..6668620f45ca 100644 --- a/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/test_k3_decode_worker.py +++ b/tests/unittest/_torch/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/test_k3_decode_worker.py @@ -29,6 +29,7 @@ import pytest import torch +from _torch.helpers import give_stub_a_draft from torch import nn from tensorrt_llm._device_copy import DeviceCopyBuffer @@ -137,9 +138,9 @@ def _accept_case(): num_seqs=NUM_GENS, kv_cache_manager=_kda_manager(), mamba_metadata=SimpleNamespace(state_indices=torch.zeros(4, dtype=torch.int32)), - draft_kv_cache_block_offsets=torch.zeros(1, 4, 2, 3, dtype=torch.int32), kv_lens_cuda=torch.zeros(4, dtype=torch.int32), ) + give_stub_a_draft(attn, torch.zeros(1, 4, 2, 3, dtype=torch.int32)) worker = _worker( k3_decode=True, mapping=TP4, @@ -240,7 +241,7 @@ def mutate(case): ), "the private context arena": _set("worker._ctx_block_tables", None), "a sliding-window draft pool": _set("worker._ctx_pool_windowed", True), - "no draft block offsets": _set("attn.draft_kv_cache_block_offsets", None), + "no draft block offsets": _set("attn.draft.kv_cache_block_offsets", None), "no KV lengths": _set("attn.kv_lens_cuda", None), "a vocab-sharded draft embedding": _set( "drafter.draft_model_full.model.embed_tokens.tp_size", 2 @@ -707,13 +708,12 @@ def test_dspark_acceptance_is_kept_only_under_k3_decode(monkeypatch): on.sample_and_accept_draft_tokens(None, attn, spec) kept = on._k3_acceptance assert kept[0] is accepted and kept[1] is num_accepted and kept[2] == 0 - assert kept[3] is spec and kept[4] is attn + assert kept[3] is spec -@pytest.mark.parametrize("pending", [True, False], ids=["a pending KV rewind", "no pending rewind"]) -def test_k3_markov_chain_drafts_and_next_new_tokens(monkeypatch, pending): - """``k3_markov`` gets this rank's slice and the step's acceptance, folds a pending KV-length rewind, and its - tokens and next_new_tokens are the step's drafts and next inputs.""" +def test_k3_markov_chain_drafts_and_next_new_tokens(monkeypatch): + """``k3_markov`` gets this rank's slice and the step's acceptance, and its tokens and next_new_tokens are the + step's drafts and next inputs. It rewinds no KV lengths: the draft loop advanced the draft metadata's only.""" worker = _worker(KimiK3DSparkWorker, k3_decode=True, mapping=TP4) accepted = torch.zeros(NUM_GENS, K + 1, dtype=torch.int32) num_accepted = torch.ones(NUM_GENS, dtype=torch.int32) @@ -722,9 +722,6 @@ def test_k3_markov_chain_drafts_and_next_new_tokens(monkeypatch, pending): ) attn = SimpleNamespace(num_contexts=0, kv_lens_cuda=torch.zeros(4, dtype=torch.int32)) worker._on_acceptance(accepted, num_accepted, attn, spec) - rewind = torch.zeros(NUM_GENS, dtype=torch.int32) - worker._kv_rewind_amount, worker._kv_rewind_pending = rewind, pending - worker._kv_rewind_nc, worker._kv_rewind_bs = 0, NUM_GENS outputs = ( torch.zeros(NUM_GENS, K, SHARD), torch.zeros(NUM_GENS, K, dtype=torch.int32), @@ -749,13 +746,7 @@ def test_k3_markov_chain_drafts_and_next_new_tokens(monkeypatch, pending): assert torch.equal(args[4], drafter.markov_w2[vocab_slice]) and args[5] == vocab_slice.start assert args[6] is accepted and torch.equal(args[7], num_accepted) assert torch.equal(args[8], spec.batch_indices_cuda[:NUM_GENS]) - if pending: - assert kwargs["kv_lens"] is attn.kv_lens_cuda and kwargs["rewind"] is rewind - assert worker._kv_rewind_amount is None and not worker._kv_rewind_pending - else: - assert kwargs["kv_lens"] is None and kwargs["rewind"] is None - assert worker._kv_rewind_amount is rewind - assert kwargs["rewind_first"] == 0 + assert "kv_lens" not in kwargs and "rewind" not in kwargs drafts = worker.sample_draft_tokens(corrected, spec, NUM_GENS, num_contexts=0) assert drafts is outputs[1] diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py index 7bfd8de384f9..28ad875449e9 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py @@ -172,7 +172,7 @@ def make_msa_metadata(manager) -> MiniMaxM3MsaSparseAttentionMetadata: def make_draft_metadata(manager) -> TrtllmAttentionMetadata: - """Drafter metadata over the shared manager's FP8 draft view, the manager the runtime's draft swap installs.""" + """Drafter metadata over the shared manager's FP8 draft view, the manager a draft metadata runs on.""" view = manager.get_draft_kv_cache_view() assert view is not None and view.dtype == DataType.FP8 return TrtllmAttentionMetadata( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_modeling_v2_m3_dispatch.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_modeling_v2_m3_dispatch.py index 450acab7c7d4..280b7b5fb885 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_modeling_v2_m3_dispatch.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/test_modeling_v2_m3_dispatch.py @@ -311,7 +311,7 @@ def test_the_core_selects_the_target_on_the_context_count_alone() -> None: "msa_decode_span", "msa_block_table", "msa_seq_lens_cuda", - "_msa_short_ok", + "msa_short_table_staged", "msa_short_table", "msa_proxy_max_score_view", "msa_worst_case_max_k_tiles", @@ -323,6 +323,7 @@ def _decode_metadata(num_tokens: int) -> SimpleNamespace: """The decode projection's fields and nothing else (no ``num_contexts`` / ``num_ctx_tokens``).""" md = SimpleNamespace(**{name: object() for name in _DECODE_FIELDS}) md.num_tokens = num_tokens + md.msa_write_slots = lambda n=None: md.msa_out_cache_loc return md @@ -330,7 +331,7 @@ def test_the_decode_projection_states_the_phase_and_reads_no_count_back() -> Non md = _decode_metadata(4) step = dispatch.DecodeTarget(SimpleNamespace()).step_args(md) assert step["num_contexts"] == 0 and step["num_ctx_tokens"] == 0 - assert step["metadata"] is md and step["msa_short_ok"] is md._msa_short_ok + assert step["metadata"] is md and step["msa_short_ok"] is md.msa_short_table_staged @pytest.mark.parametrize( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py index e9aaad5c65fa..a3c987b0453f 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py @@ -172,7 +172,7 @@ def make_msa_metadata(manager) -> MiniMaxM3MsaSparseAttentionMetadata: def make_draft_metadata(manager) -> TrtllmAttentionMetadata: - """Drafter metadata over the shared manager's FP8 draft view, the manager the runtime's draft swap installs.""" + """Drafter metadata over the shared manager's FP8 draft view, the manager a draft metadata runs on.""" view = manager.get_draft_kv_cache_view() assert view is not None and view.dtype == DataType.FP8 return TrtllmAttentionMetadata( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_modeling_v2_m3_dispatch.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_modeling_v2_m3_dispatch.py index 61254b231485..81da74fa8b48 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_modeling_v2_m3_dispatch.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/test_modeling_v2_m3_dispatch.py @@ -412,7 +412,7 @@ def test_the_core_selects_the_target_on_the_context_count_alone() -> None: "msa_decode_span", "msa_block_table", "msa_seq_lens_cuda", - "_msa_short_ok", + "msa_short_table_staged", "msa_short_table", "msa_proxy_max_score_view", "msa_worst_case_max_k_tiles", @@ -424,6 +424,7 @@ def _decode_metadata(num_tokens: int) -> SimpleNamespace: """The decode projection's fields and nothing else (no ``num_contexts`` / ``num_ctx_tokens``).""" md = SimpleNamespace(**{name: object() for name in _DECODE_FIELDS}) md.num_tokens = num_tokens + md.msa_write_slots = lambda n=None: md.msa_out_cache_loc return md @@ -431,7 +432,7 @@ def test_the_decode_projection_states_the_phase_and_reads_no_count_back() -> Non md = _decode_metadata(4) step = dispatch.DecodeTarget(SimpleNamespace()).step_args(md) assert step["num_contexts"] == 0 and step["num_ctx_tokens"] == 0 - assert step["metadata"] is md and step["msa_short_ok"] is md._msa_short_ok + assert step["metadata"] is md and step["msa_short_ok"] is md.msa_short_table_staged @pytest.mark.parametrize( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py index 137230cd0345..78195491ca46 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py @@ -173,7 +173,7 @@ def make_msa_metadata(manager) -> MiniMaxM3MsaSparseAttentionMetadata: def make_draft_metadata(manager) -> TrtllmAttentionMetadata: - """Drafter metadata over the shared manager's FP8 draft view, the manager the runtime's draft swap installs.""" + """Drafter metadata over the shared manager's FP8 draft view, the manager a draft metadata runs on.""" view = manager.get_draft_kv_cache_view() assert view is not None and view.dtype == DataType.FP8 return TrtllmAttentionMetadata( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_block_table_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_block_table_contract.py index 2dfd7aa71ebb..e1399ed7e979 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_block_table_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_block_table_contract.py @@ -8,7 +8,7 @@ ``_build_msa_fields`` (called from ``prepare()``, before the replay), besides the allocation. This test scans ``tensorrt_llm/_torch`` and fails on any other write site, so a new in-graph writer has to revisit the early issue. -(``msa_seq_lens_cuda`` IS patched in-graph by ``on_update_kv_lens``; the scorer reads it before the wait only as a hint +(``msa_seq_lens_cuda`` IS patched by ``on_update_kv_lens``; the scorer reads it before the wait only as a hint and bounds every load and store by the value it reads after the wait.) CPU only: it parses the sources. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py index b883a15acd38..342580bb875a 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,7 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the runner calls before the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before + the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/spec/test_modeling_v2_m3_spec_offsets.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/spec/test_modeling_v2_m3_spec_offsets.py deleted file mode 100644 index 2e39c56397c9..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/spec/test_modeling_v2_m3_spec_offsets.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""GPU test for m3_spec_offsets (RuntimeHooks.spec_offsets_op): the overlap scheduler's two speculative step corrections -in one plain launch, bit for bit the in-place ATen adds, eager and in a CUDA graph.""" - -import pytest -import torch - -from tensorrt_llm._torch._experimental.modeling_v2.models.minimax_m3.minimax_m3_nvfp4__sm_107__tp2.ops.spec.m3_spec_offsets import ( # noqa: E501 - m3_spec_offsets as op, -) - -if not torch.cuda.is_available(): - pytest.skip("m3_spec_offsets requires a CUDA device", allow_module_level=True) - -I32 = torch.iinfo(torch.int32) - - -def _inputs(tokens, seqs, seed, wrap=False): - g = torch.Generator(device="cuda").manual_seed(seed) - lo, hi = (I32.min, I32.max) if wrap else (0, 1 << 20) - pos = torch.randint(lo, hi, (tokens,), generator=g, device="cuda", dtype=torch.int32) - kv = torch.randint(lo, hi, (seqs,), generator=g, device="cuda", dtype=torch.int32) - pos_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (tokens,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - kv_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (seqs,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - if wrap: - pos[:2], pos_off[:2] = I32.max, 1 # overflow wraps, as the ATen add does - kv[:1], kv_off[:1] = I32.min, -1 - return pos, pos_off, kv, kv_off - - -CELLS = [c.spec for c in op.CELLS] + [ - dict(tokens=1, seqs=1), - dict(tokens=300, seqs=75), - dict(tokens=4, seqs=0), -] - - -@pytest.mark.parametrize("spec", CELLS, ids=lambda s: "-".join(f"{k}{v}" for k, v in s.items())) -def test_bitwise_aten_adds(spec) -> None: - for trial in range(3): - pos, pos_off, kv, kv_off = _inputs( - spec["tokens"], spec["seqs"], 97 * trial + spec["tokens"], spec.get("wrap", False) - ) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - assert op(pos, pos_off, kv, kv_off) is True - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"trial {trial}" - - -def test_graph_replays_read_the_live_offsets() -> None: - pos, pos_off, kv, kv_off = _inputs(32, 8, 5) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - for step in range(3): - fresh = _inputs(32, 8, 11 + step) - for dst, src in zip((pos, pos_off, kv, kv_off), fresh): - dst.copy_(src) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - graph.replay() - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"step {step}" - - -def test_runner_views_are_accepted() -> None: - """The runner passes position_ids[0, ctx:] and kv_lens_cuda[ctx_reqs:seqs]: 1-D slices of contiguous buffers.""" - position_ids = torch.arange(40, dtype=torch.int32, device="cuda").view(1, 40) - kv_lens = torch.arange(12, dtype=torch.int32, device="cuda") - pos_off = torch.full((40,), 2, dtype=torch.int32, device="cuda") - kv_off = torch.full((12,), 3, dtype=torch.int32, device="cuda") - assert op(position_ids[0, 8:], pos_off[:32], kv_lens[2:10], kv_off[:8]) is True - torch.cuda.synchronize() - assert torch.equal( - position_ids[0], torch.cat([torch.arange(8), torch.arange(8, 40) + 2]).int().cuda() - ) - assert torch.equal( - kv_lens, - torch.cat([torch.arange(2), torch.arange(2, 10) + 3, torch.arange(10, 12)]).int().cuda(), - ) - - -@pytest.mark.parametrize( - "bad", - ["int64", "strided", "length", "2d"], -) -def test_declines_other_layouts_without_writing(bad) -> None: - pos, pos_off, kv, kv_off = _inputs(8, 2, 3) - if bad == "int64": - kv = kv.long() - elif bad == "strided": - pos = torch.zeros(16, dtype=torch.int32, device="cuda")[::2] - elif bad == "length": - pos_off = pos_off[:7] - else: - pos = pos.view(2, 4) - before = (pos.clone(), kv.clone()) - assert op(pos, pos_off, kv, kv_off) is False - torch.cuda.synchronize() - assert torch.equal(pos, before[0]) and torch.equal(kv, before[1]) - - -def test_launches_plainly() -> None: - """It writes step inputs that pre-wait readers take before their grid wait (MSA contract, dispatch - PRODUCER_PREWAIT_STEP_INPUTS): its kernel node carries no programmatic (PDL) serialization.""" - from cuda.bindings import runtime as rt - - pos, pos_off, kv, kv_off = _inputs(32, 8, 7) - graph = torch.cuda.CUDAGraph(keep_graph=True) - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - raw = rt.cudaGraph_t(init_value=graph.raw_cuda_graph()) - err, _, n = rt.cudaGraphGetNodes(raw, 0) - err, nodes, n = rt.cudaGraphGetNodes(raw, n) - flags = [] - for node in nodes[:n]: - err, kind = rt.cudaGraphNodeGetType(node) - if kind == rt.cudaGraphNodeType.cudaGraphNodeTypeKernel: - err, attr = rt.cudaGraphKernelNodeGetAttribute( - node, rt.cudaLaunchAttributeID.cudaLaunchAttributeProgrammaticStreamSerialization - ) - flags.append(0 if attr is None else int(attr.programmaticStreamSerializationAllowed)) - assert flags == [0], flags diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py index 7bfd8de384f9..28ad875449e9 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py @@ -172,7 +172,7 @@ def make_msa_metadata(manager) -> MiniMaxM3MsaSparseAttentionMetadata: def make_draft_metadata(manager) -> TrtllmAttentionMetadata: - """Drafter metadata over the shared manager's FP8 draft view, the manager the runtime's draft swap installs.""" + """Drafter metadata over the shared manager's FP8 draft view, the manager a draft metadata runs on.""" view = manager.get_draft_kv_cache_view() assert view is not None and view.dtype == DataType.FP8 return TrtllmAttentionMetadata( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py index be10d54aca2a..03df0439f559 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,7 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the runner calls before the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before + the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/spec/test_modeling_v2_m3_spec_offsets.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/spec/test_modeling_v2_m3_spec_offsets.py deleted file mode 100644 index 74e63d9e8e5e..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/spec/test_modeling_v2_m3_spec_offsets.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""GPU test for m3_spec_offsets (RuntimeHooks.spec_offsets_op): the overlap scheduler's two speculative step corrections -in one plain launch, bit for bit the in-place ATen adds, eager and in a CUDA graph.""" - -import pytest -import torch - -from tensorrt_llm._torch._experimental.modeling_v2.models.minimax_m3.minimax_m3_nvfp4__sm_107__tp4.ops.spec.m3_spec_offsets import ( # noqa: E501 - m3_spec_offsets as op, -) - -if not torch.cuda.is_available(): - pytest.skip("m3_spec_offsets requires a CUDA device", allow_module_level=True) - -I32 = torch.iinfo(torch.int32) - - -def _inputs(tokens, seqs, seed, wrap=False): - g = torch.Generator(device="cuda").manual_seed(seed) - lo, hi = (I32.min, I32.max) if wrap else (0, 1 << 20) - pos = torch.randint(lo, hi, (tokens,), generator=g, device="cuda", dtype=torch.int32) - kv = torch.randint(lo, hi, (seqs,), generator=g, device="cuda", dtype=torch.int32) - pos_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (tokens,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - kv_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (seqs,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - if wrap: - pos[:2], pos_off[:2] = I32.max, 1 # overflow wraps, as the ATen add does - kv[:1], kv_off[:1] = I32.min, -1 - return pos, pos_off, kv, kv_off - - -CELLS = [c.spec for c in op.CELLS] + [ - dict(tokens=1, seqs=1), - dict(tokens=300, seqs=75), - dict(tokens=4, seqs=0), -] - - -@pytest.mark.parametrize("spec", CELLS, ids=lambda s: "-".join(f"{k}{v}" for k, v in s.items())) -def test_bitwise_aten_adds(spec) -> None: - for trial in range(3): - pos, pos_off, kv, kv_off = _inputs( - spec["tokens"], spec["seqs"], 97 * trial + spec["tokens"], spec.get("wrap", False) - ) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - assert op(pos, pos_off, kv, kv_off) is True - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"trial {trial}" - - -def test_graph_replays_read_the_live_offsets() -> None: - pos, pos_off, kv, kv_off = _inputs(32, 8, 5) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - for step in range(3): - fresh = _inputs(32, 8, 11 + step) - for dst, src in zip((pos, pos_off, kv, kv_off), fresh): - dst.copy_(src) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - graph.replay() - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"step {step}" - - -def test_runner_views_are_accepted() -> None: - """The runner passes position_ids[0, ctx:] and kv_lens_cuda[ctx_reqs:seqs]: 1-D slices of contiguous buffers.""" - position_ids = torch.arange(40, dtype=torch.int32, device="cuda").view(1, 40) - kv_lens = torch.arange(12, dtype=torch.int32, device="cuda") - pos_off = torch.full((40,), 2, dtype=torch.int32, device="cuda") - kv_off = torch.full((12,), 3, dtype=torch.int32, device="cuda") - assert op(position_ids[0, 8:], pos_off[:32], kv_lens[2:10], kv_off[:8]) is True - torch.cuda.synchronize() - assert torch.equal( - position_ids[0], torch.cat([torch.arange(8), torch.arange(8, 40) + 2]).int().cuda() - ) - assert torch.equal( - kv_lens, - torch.cat([torch.arange(2), torch.arange(2, 10) + 3, torch.arange(10, 12)]).int().cuda(), - ) - - -@pytest.mark.parametrize( - "bad", - ["int64", "strided", "length", "2d"], -) -def test_declines_other_layouts_without_writing(bad) -> None: - pos, pos_off, kv, kv_off = _inputs(8, 2, 3) - if bad == "int64": - kv = kv.long() - elif bad == "strided": - pos = torch.zeros(16, dtype=torch.int32, device="cuda")[::2] - elif bad == "length": - pos_off = pos_off[:7] - else: - pos = pos.view(2, 4) - before = (pos.clone(), kv.clone()) - assert op(pos, pos_off, kv, kv_off) is False - torch.cuda.synchronize() - assert torch.equal(pos, before[0]) and torch.equal(kv, before[1]) - - -def test_launches_plainly() -> None: - """It writes step inputs that pre-wait readers take before their grid wait (MSA contract, dispatch - PRODUCER_PREWAIT_STEP_INPUTS): its kernel node carries no programmatic (PDL) serialization.""" - from cuda.bindings import runtime as rt - - pos, pos_off, kv, kv_off = _inputs(32, 8, 7) - graph = torch.cuda.CUDAGraph(keep_graph=True) - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - raw = rt.cudaGraph_t(init_value=graph.raw_cuda_graph()) - err, _, n = rt.cudaGraphGetNodes(raw, 0) - err, nodes, n = rt.cudaGraphGetNodes(raw, n) - flags = [] - for node in nodes[:n]: - err, kind = rt.cudaGraphNodeGetType(node) - if kind == rt.cudaGraphNodeType.cudaGraphNodeTypeKernel: - err, attr = rt.cudaGraphKernelNodeGetAttribute( - node, rt.cudaLaunchAttributeID.cudaLaunchAttributeProgrammaticStreamSerialization - ) - flags.append(0 if attr is None else int(attr.programmaticStreamSerializationAllowed)) - assert flags == [0], flags diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py index e9aaad5c65fa..a3c987b0453f 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py @@ -172,7 +172,7 @@ def make_msa_metadata(manager) -> MiniMaxM3MsaSparseAttentionMetadata: def make_draft_metadata(manager) -> TrtllmAttentionMetadata: - """Drafter metadata over the shared manager's FP8 draft view, the manager the runtime's draft swap installs.""" + """Drafter metadata over the shared manager's FP8 draft view, the manager a draft metadata runs on.""" view = manager.get_draft_kv_cache_view() assert view is not None and view.dtype == DataType.FP8 return TrtllmAttentionMetadata( diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py index b883a15acd38..342580bb875a 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,7 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the runner calls before the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before + the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/spec/test_modeling_v2_m3_spec_offsets.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/spec/test_modeling_v2_m3_spec_offsets.py deleted file mode 100644 index 2e8b405a4e0b..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/spec/test_modeling_v2_m3_spec_offsets.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""GPU test for m3_spec_offsets (RuntimeHooks.spec_offsets_op): the overlap scheduler's two speculative step corrections -in one plain launch, bit for bit the in-place ATen adds, eager and in a CUDA graph.""" - -import pytest -import torch - -from tensorrt_llm._torch._experimental.modeling_v2.models.minimax_m3.minimax_m3_nvfp4__sm_107__tp8.ops.spec.m3_spec_offsets import ( # noqa: E501 - m3_spec_offsets as op, -) - -if not torch.cuda.is_available(): - pytest.skip("m3_spec_offsets requires a CUDA device", allow_module_level=True) - -I32 = torch.iinfo(torch.int32) - - -def _inputs(tokens, seqs, seed, wrap=False): - g = torch.Generator(device="cuda").manual_seed(seed) - lo, hi = (I32.min, I32.max) if wrap else (0, 1 << 20) - pos = torch.randint(lo, hi, (tokens,), generator=g, device="cuda", dtype=torch.int32) - kv = torch.randint(lo, hi, (seqs,), generator=g, device="cuda", dtype=torch.int32) - pos_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (tokens,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - kv_off = torch.randint( - -8 if not wrap else lo, - 8 if not wrap else hi, - (seqs,), - generator=g, - device="cuda", - dtype=torch.int32, - ) - if wrap: - pos[:2], pos_off[:2] = I32.max, 1 # overflow wraps, as the ATen add does - kv[:1], kv_off[:1] = I32.min, -1 - return pos, pos_off, kv, kv_off - - -CELLS = [c.spec for c in op.CELLS] + [ - dict(tokens=1, seqs=1), - dict(tokens=300, seqs=75), - dict(tokens=4, seqs=0), -] - - -@pytest.mark.parametrize("spec", CELLS, ids=lambda s: "-".join(f"{k}{v}" for k, v in s.items())) -def test_bitwise_aten_adds(spec) -> None: - for trial in range(3): - pos, pos_off, kv, kv_off = _inputs( - spec["tokens"], spec["seqs"], 97 * trial + spec["tokens"], spec.get("wrap", False) - ) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - assert op(pos, pos_off, kv, kv_off) is True - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"trial {trial}" - - -def test_graph_replays_read_the_live_offsets() -> None: - pos, pos_off, kv, kv_off = _inputs(32, 8, 5) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - for step in range(3): - fresh = _inputs(32, 8, 11 + step) - for dst, src in zip((pos, pos_off, kv, kv_off), fresh): - dst.copy_(src) - ref_pos, ref_kv = pos.clone(), kv.clone() - op.reference(ref_pos, pos_off, ref_kv, kv_off) - graph.replay() - torch.cuda.synchronize() - assert torch.equal(pos, ref_pos) and torch.equal(kv, ref_kv), f"step {step}" - - -def test_runner_views_are_accepted() -> None: - """The runner passes position_ids[0, ctx:] and kv_lens_cuda[ctx_reqs:seqs]: 1-D slices of contiguous buffers.""" - position_ids = torch.arange(40, dtype=torch.int32, device="cuda").view(1, 40) - kv_lens = torch.arange(12, dtype=torch.int32, device="cuda") - pos_off = torch.full((40,), 2, dtype=torch.int32, device="cuda") - kv_off = torch.full((12,), 3, dtype=torch.int32, device="cuda") - assert op(position_ids[0, 8:], pos_off[:32], kv_lens[2:10], kv_off[:8]) is True - torch.cuda.synchronize() - assert torch.equal( - position_ids[0], torch.cat([torch.arange(8), torch.arange(8, 40) + 2]).int().cuda() - ) - assert torch.equal( - kv_lens, - torch.cat([torch.arange(2), torch.arange(2, 10) + 3, torch.arange(10, 12)]).int().cuda(), - ) - - -@pytest.mark.parametrize( - "bad", - ["int64", "strided", "length", "2d"], -) -def test_declines_other_layouts_without_writing(bad) -> None: - pos, pos_off, kv, kv_off = _inputs(8, 2, 3) - if bad == "int64": - kv = kv.long() - elif bad == "strided": - pos = torch.zeros(16, dtype=torch.int32, device="cuda")[::2] - elif bad == "length": - pos_off = pos_off[:7] - else: - pos = pos.view(2, 4) - before = (pos.clone(), kv.clone()) - assert op(pos, pos_off, kv, kv_off) is False - torch.cuda.synchronize() - assert torch.equal(pos, before[0]) and torch.equal(kv, before[1]) - - -def test_launches_plainly() -> None: - """It writes step inputs that pre-wait readers take before their grid wait (MSA contract, dispatch - PRODUCER_PREWAIT_STEP_INPUTS): its kernel node carries no programmatic (PDL) serialization.""" - from cuda.bindings import runtime as rt - - pos, pos_off, kv, kv_off = _inputs(32, 8, 7) - graph = torch.cuda.CUDAGraph(keep_graph=True) - with torch.cuda.graph(graph): - op(pos, pos_off, kv, kv_off) - raw = rt.cudaGraph_t(init_value=graph.raw_cuda_graph()) - err, _, n = rt.cudaGraphGetNodes(raw, 0) - err, nodes, n = rt.cudaGraphGetNodes(raw, n) - flags = [] - for node in nodes[:n]: - err, kind = rt.cudaGraphNodeGetType(node) - if kind == rt.cudaGraphNodeType.cudaGraphNodeTypeKernel: - err, attr = rt.cudaGraphKernelNodeGetAttribute( - node, rt.cudaLaunchAttributeID.cudaLaunchAttributeProgrammaticStreamSerialization - ) - flags.append(0 if attr is None else int(attr.programmaticStreamSerializationAllowed)) - assert flags == [0], flags diff --git a/tests/unittest/_torch/models/test_minimax_m3.py b/tests/unittest/_torch/models/test_minimax_m3.py index 67af020c53a5..a72149922098 100644 --- a/tests/unittest/_torch/models/test_minimax_m3.py +++ b/tests/unittest/_torch/models/test_minimax_m3.py @@ -28,6 +28,7 @@ import pytest import torch +from _torch.helpers import bare_extension from safetensors import safe_open from torch import nn from torch._dynamo.backends.common import aot_autograd @@ -40,6 +41,7 @@ from utils.util import skip_blackwell_geforce, skip_pre_blackwell import tensorrt_llm._torch.models.modeling_minimaxm3 as modeling_minimaxm3 +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.fmha.msa_prefill import _aligned_nvfp4_dequant_scales from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import ( @@ -55,9 +57,6 @@ MiniMaxM3SparseParams, index_head_range, ) -from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaSparseAttentionMetadata, -) from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_indexer import _group_max_reduce from tensorrt_llm._torch.compilation.remove_copy_pass import remove_copy_for_mutates_args from tensorrt_llm._torch.model_config import ModelConfig @@ -737,11 +736,11 @@ def test_piecewise_captures_horizontal_producer_before_attention( scale_cache = torch.empty(8, dtype=torch.uint8) if kv_dtype == "nvfp4" else None slots = torch.tensor([0, 1, -1, -1], dtype=torch.int32) metadata = Mock( - spec_set=MiniMaxM3MsaSparseAttentionMetadata, num_tokens=2, kv_cache_manager=None, msa_layer_cache_tensors={3: (kv_cache, index_cache, scale_cache)}, msa_out_cache_loc=slots, + msa_write_slots=Mock(return_value=slots), ) layer = SimpleNamespace( enable_fused_qkv_index_projection=True, @@ -991,7 +990,6 @@ def test_piecewise_attention_boundary_preserves_indexer_cache_contract( index_cache.copy_(expected_cache) selected_blocks = torch.zeros(2, 1, 16, dtype=torch.int32) attn_metadata = Mock( - spec_set=MiniMaxM3MsaSparseAttentionMetadata, num_tokens=2, kv_cache_manager=None, msa_decode_span=None, @@ -999,7 +997,7 @@ def test_piecewise_attention_boundary_preserves_indexer_cache_contract( msa_write_idx_k=Mock(), msa_prefill_proxy_plan=None, msa_prefill_n_valid_blocks=None, - _msa_prefill_identity=None, + msa_prefill_identity=None, msa_prefill_proxy_kv_indices=torch.tensor([0, 1], dtype=torch.int32), msa_qo_lens_cpu=torch.tensor([2], dtype=torch.int32), msa_kv_lens_cpu=torch.tensor([2], dtype=torch.int32), @@ -2219,9 +2217,9 @@ def test_horizontal_producer_explicit_caches_match_manager( manager.get_buffers.return_value = main manager.get_block_scale_buffers.return_value = scales metadata = Mock( - spec_set=MiniMaxM3MsaSparseAttentionMetadata, kv_cache_manager=manager, msa_out_cache_loc=slots, + msa_write_slots=Mock(side_effect=lambda n=None: slots if n is None else slots[:n]), ) metadata.msa_idx_k_cache.return_value = index @@ -3356,9 +3354,7 @@ def test_minimax_m3_dense_backend_library_order( metadata.cu_q_seqlens = metadata.cu_kv_seqlens = torch.tensor([0, 2], dtype=torch.int32) metadata.num_contexts = 1 metadata.num_generations = 0 - metadata.enable_flash_mla = False - metadata.spec_bl_tree_first_sparse_mask_offset_kv = None - metadata.spec_decoding_bl_tree_mask = None + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata.max_context_q_len_override = None metadata.kv_cache_manager = SimpleNamespace( dtype=DataType.BF16, max_attention_window_vec=[128], max_seq_len=128 @@ -3840,10 +3836,12 @@ def test_minimax_m3_sdpa_dense_decode_matches_repeat_interleave_reference(num_kv ) attn_metadata = SimpleNamespace( kv_cache_manager=SimpleNamespace(get_buffers=lambda layer_idx: kv_pool), - minimax_m3={"metadata": m3_meta, "out_cache_loc": out_cache_loc}, + sparse_metadata=m3_meta, + out_cache_loc=out_cache_loc, is_cuda_graph=False, max_seq_len=max_len, ) + attn_metadata.find_ext = lambda cls: attn_metadata output = torch.empty(batch, num_heads * head_dim, device=device, dtype=dtype) MiniMaxM3Attention._sdpa_dense_attention_core(layer, q, k_new, v_new, attn_metadata, output) diff --git a/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py b/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py index ee84456441aa..b7f7ecf1c391 100644 --- a/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py +++ b/tests/unittest/_torch/modules/mamba/test_flashinfer_chunk_gdn.py @@ -289,16 +289,17 @@ def test_state_workspace_warmup_and_reuse( import tensorrt_llm._torch.modules.mamba.gdn_mixer as mixer from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata - from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata + from tensorrt_llm._torch.attention.backends.trtllm import ( + TrtllmAttentionMetadata, + TrtllmCoreMetadata, + ) monkeypatch.setattr(mixer, "_use_flashinfer_gdn_prefill", lambda: True) monkeypatch.setattr(mixer, "is_sm_100f", lambda: native_state) config = SimpleNamespace(extra_attrs={}) layer = SimpleNamespace(model_config=config) # Keep the real metadata contract; bypass only TRTLLM's unrelated CUDA buffers. - monkeypatch.setattr( - TrtllmAttentionMetadata, "_post_init_with_buffers", lambda self, buffers: None - ) + monkeypatch.setattr(TrtllmCoreMetadata, "_allocate_buffers", lambda self, buffers: None) metadata_cls = AttentionMetadata if metadata_backend == "base" else TrtllmAttentionMetadata metadata = metadata_cls( max_num_sequences=max_sequences, diff --git a/tests/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py b/tests/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py index 44bed8a2b390..912aab3f478a 100644 --- a/tests/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py +++ b/tests/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py @@ -526,10 +526,10 @@ def _check_dp_replay_case( assert plan is not None kept = expected_counts[rank] start = count - kept - new_positions, new_ids, replay_state = model._enter_bounded_replay( + new_positions, new_ids, replay_state, layer_metadata = model._enter_bounded_replay( plan, metadata, positions, input_ids, state ) - assert metadata.all_rank_num_tokens == expected_counts + assert layer_metadata.all_rank_num_tokens == expected_counts torch.testing.assert_close(new_positions, positions[start:], rtol=0, atol=0) torch.testing.assert_close(new_ids, input_ids[start:], rtol=0, atol=0) if start == 0: @@ -537,9 +537,11 @@ def _check_dp_replay_case( assert replay_state.residual is state.residual if not plan.updates_local_metadata: assert replay_state is state + assert layer_metadata is metadata assert metadata.csa2_replay_query_rows is None else: - assert metadata.csa2_replay_query_rows is plan.rows + assert layer_metadata is metadata.decoder_replay_metadata + assert layer_metadata.csa2_replay_query_rows is plan.rows # Full-prefill parity is invalid for this approximation. Build the same # bounded replay independently of the DP planner, retaining all GLOBAL KV. @@ -553,7 +555,7 @@ def _check_dp_replay_case( reference_metadata.csa2_precomputed_kv_layers = {0} if kept: actual_attention = model.layers[0].self_attn( - norm(replay_state.residual[:, 0]), new_positions, metadata + norm(replay_state.residual[:, 0]), new_positions, layer_metadata ) expected_attention = reference_attention( norm(hidden[start:]), reference_metadata.csa2_positions, reference_metadata @@ -564,7 +566,9 @@ def _check_dp_replay_case( logits = torch.arange(8, device="cuda", dtype=torch.float32)[None, :].expand(kept, -1) logits = torch.sin(logits + new_positions[:, None].float() + rank) if any(expected_counts): - actual = moe(actual_attention, logits, all_rank_num_tokens=metadata.all_rank_num_tokens) + actual = moe( + actual_attention, logits, all_rank_num_tokens=layer_metadata.all_rank_num_tokens + ) expected = reference_moe(expected_attention, logits) else: actual = expected = hidden[:0] @@ -726,12 +730,12 @@ def run(requests: list[LlmRequest], frozen: list[dict] | None = None) -> None: step_metadata.kv_cache_params = KVCacheParams( use_cache=True, num_cached_tokens_per_seq=[start] ) - if step_metadata is metadata: - step_metadata.prepare_context_replay([step_request]) - elif encoder_replay_tokens(step_request): + if step_metadata is not metadata and encoder_replay_tokens(step_request): step_metadata.set_swa_bounded_replay([step_request.context_current_position]) - step_metadata.prepare() - step_metadata.begin_model_forward() + step_metadata.prepare( + context_requests=[step_request] if step_metadata is metadata else None + ) + step_metadata.reset_forward_state() check_floor(step_metadata, 0, start) metadata.all_rank_num_tokens = counts assert MPI.COMM_WORLD.allgather(metadata.num_tokens) == counts @@ -743,16 +747,21 @@ def run(requests: list[LlmRequest], frozen: list[dict] | None = None) -> None: assert plan is not None def layer_output( - layer: int, actual_hidden: torch.Tensor, expected_hidden: torch.Tensor + layer: int, + actual_hidden: torch.Tensor, + expected_hidden: torch.Tensor, + layer_metadata: CSA2TrtllmMetadata = metadata, ) -> torch.Tensor: - actual = attention[layer](norm(actual_hidden), metadata.csa2_positions, metadata) + actual = attention[layer]( + norm(actual_hidden), layer_metadata.csa2_positions, layer_metadata + ) expected = reference_attention[layer]( norm(expected_hidden), reference_metadata.csa2_positions, reference_metadata ) torch.testing.assert_close(actual, expected, rtol=0, atol=0) logits = torch.arange(8, device="cuda", dtype=torch.float32)[None, :] - logits = torch.sin(logits + metadata.csa2_positions[:, None].float() + rank) - output = moe(actual, logits, all_rank_num_tokens=metadata.all_rank_num_tokens) + logits = torch.sin(logits + layer_metadata.csa2_positions[:, None].float() + rank) + output = moe(actual, logits, all_rank_num_tokens=layer_metadata.all_rank_num_tokens) reference = reference_moe(expected, logits) assert torch.isfinite(output).all() reference_moe.check_accuracy(output, reference) @@ -768,24 +777,24 @@ def layer_output( reference_metadata.seq_lens = torch.tensor([kept], dtype=torch.int32, device="cpu") reference_metadata.prompt_lens = [kept] reference_metadata.kv_cache_params.num_cached_tokens_per_seq = [decoder_start] - reference_metadata.set_decoder_query_boundary(1) + reference_metadata.set_decoder_query_boundary(1, [decoder_start + kept]) reference_metadata.prepare() reference_metadata.csa2_precomputed_kv_layers = {1} reference_metadata._ensure_swa_slots() logical = reference_metadata.csa2_positions[:, None] - _REPLAY_WINDOW + 1 logical = logical + torch.arange(_REPLAY_WINDOW, device="cuda") reference_metadata.csa2_swa_indices[1].masked_fill_(logical < decoder_start, -1) - new_positions, new_ids, replay_state = model._enter_bounded_replay( + new_positions, new_ids, replay_state, replay = model._enter_bounded_replay( plan, metadata, positions, ids, state ) expected_counts = [min(count, _REPLAY_WINDOW) for count in counts] - assert metadata.all_rank_num_tokens == expected_counts + assert replay.all_rank_num_tokens == expected_counts assert MPI.COMM_WORLD.allgather(replay_state.residual.shape[0]) == expected_counts torch.testing.assert_close(new_positions, positions[-kept:], rtol=0, atol=0) torch.testing.assert_close(new_ids, ids[-kept:], rtol=0, atol=0) - check_floor(metadata, 1, decoder_start) + check_floor(replay, 1, decoder_start) # Match encoder quantization so this comparison isolates the decoder cache and rows. - output = layer_output(1, replay_state.residual[:, 0], encoder[-kept:]) + output = layer_output(1, replay_state.residual[:, 0], encoder[-kept:], replay) restored = model._exit_bounded_replay(plan, metadata, output) assert restored.shape == hidden.shape assert not restored[:-kept].count_nonzero() diff --git a/tests/unittest/_torch/multi_gpu/test_minimax_m3_piecewise.py b/tests/unittest/_torch/multi_gpu/test_minimax_m3_piecewise.py index 42ca4d70539a..b20546972089 100644 --- a/tests/unittest/_torch/multi_gpu/test_minimax_m3_piecewise.py +++ b/tests/unittest/_torch/multi_gpu/test_minimax_m3_piecewise.py @@ -10,11 +10,16 @@ import cloudpickle import pytest import torch +from _torch.helpers import bare_extension from mpi4py import MPI from mpi4py.futures import MPIPoolExecutor import tensorrt_llm +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import MiniMaxM3MsaSparseAttention +from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( + MiniMaxM3MsaMetadata, +) from tensorrt_llm._torch.compilation.piecewise_optimizer import PiecewiseRunner from tensorrt_llm._torch.compilation.utils import capture_piecewise_cuda_graph from tensorrt_llm._torch.distributed import Distributed @@ -66,6 +71,8 @@ def _run_empty_adp_rank(world_size: int) -> int: # allocator is stubbed; slot construction/clearing and GPU writes are real. metadata_cls = MiniMaxM3MsaSparseAttention.Metadata metadata = metadata_cls.__new__(metadata_cls) + metadata.replace_ext(bare_extension(MiniMaxM3MsaMetadata, metadata)) + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata._msa_buffers_ready = True metadata.kv_cache_manager = SimpleNamespace( tokens_per_block=128, @@ -150,6 +157,7 @@ def producer( # read the live (unpadded) count off it. metadata._num_tokens = live_tokens metadata._build_msa_fields() + metadata.msa_write_slots() assert metadata.msa_out_cache_loc.tolist() == list( range(128, 128 + live_tokens) ) + [-1] * (bucket - live_tokens) diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_inputs.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_inputs.py index 5add16dd533a..78475e0af4d6 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_inputs.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_inputs.py @@ -515,7 +515,7 @@ def test_spec_decode_graph_step_gather_kernel_matches_torch_path(self) -> None: buffers=preparer._buffers, attention_runtime_features=AttentionRuntimeFeatures(), ).setup_attention(kv_cache_manager) - self.assertIs(type(attn_metadata), TrtllmAttentionMetadata) + self.assertIsInstance(attn_metadata, TrtllmAttentionMetadata) graph_metadata = attn_metadata.create_cuda_graph_metadata( len(requests), False, max_draft_len ) diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_metadata.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_metadata.py index 0b3f4e658e38..5f0a6957012e 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_metadata.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_metadata.py @@ -7,9 +7,11 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.pyexecutor.engine.runners.decoder import DecoderRunner from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.metadata import ( set_spec_metadata_all_rank_num_tokens, + spec_decoding_config, update_spec_metadata, ) from tensorrt_llm._torch.speculative.dspark import DSparkSpecMetadata @@ -65,14 +67,18 @@ def test_update_spec_metadata_handles_parallel_draft_and_dynamic_tree() -> None: ) attn_metadata.update_spec_dec_param.assert_called_once_with( batch_size=2, - is_spec_decoding_enabled=True, + spec_metadata=spec_metadata, + spec_tree_manager=spec_tree_manager, + num_contexts=1, + ) + spec_tree_manager._internal_buf_dim = 8 + assert spec_decoding_config(config, spec_metadata, spec_tree_manager) == SpecDecodingConfig( + attention_needs_spec_dec_mode=True, is_spec_dec_tree=True, is_spec_dec_dynamic_tree=True, max_draft_len=6, max_total_draft_tokens=6, - spec_metadata=spec_metadata, - spec_tree_manager=spec_tree_manager, - num_contexts=1, + tree_buffer_dim=8, ) diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_runner.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_runner.py index fc5cbb092b04..0627c8a1414f 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_runner.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_runner.py @@ -680,107 +680,6 @@ def _metadata() -> CSA2TrtllmMetadata: metadata.is_cuda_graph = True return metadata - def test_scope_requires_selected_ordinary_decode_graph(self) -> None: - key = KeyType(batch_size=1, draft_len=0) - cases = ( - "ordinary", - "ordinary_overlap", - "graph_miss", - "eager_metadata", - "context", - "promoted_context", - "speculative", - "beam", - "encoder_decoder", - "context_parallel", - "draft_key", - ) - for case in cases: - with self.subTest(case=case): - engine, _, _, _, _ = _make_decoder_runner( - key, - disable_overlap_scheduler=case != "ordinary_overlap", - max_beam_width=2 if case == "beam" else 1, - ) - enable_spec_decode = False - metadata = self._metadata() - batch = ScheduledRequests() - batch.generation_requests = [_make_request_stub(1)] - selected = key - promoted = frozenset() - if case == "graph_miss": - selected = None - elif case == "eager_metadata": - metadata.is_cuda_graph = False - elif case == "context": - batch.context_requests_last_chunk = [_make_request_stub(2)] - elif case == "promoted_context": - promoted = frozenset({1}) - elif case == "speculative": - enable_spec_decode = True - elif case == "encoder_decoder": - engine._deferred_decode_outputs_supported = False - elif case == "context_parallel": - engine._mapping.cp_size = 2 - elif case == "draft_key": - selected = key._replace(draft_len=1) - with engine._cuda_graph_metadata_prepare_scope( - metadata, selected, batch, promoted, enable_spec_decode=enable_spec_decode - ): - self.assertEqual( - metadata._csa2_defer_decode_outputs, - case in ("ordinary", "ordinary_overlap"), - ) - self.assertFalse(metadata._csa2_defer_decode_outputs) - - def test_forward_scopes_proof_to_prepare_for_capture_replay_and_fallback(self) -> None: - key = KeyType(batch_size=1, draft_len=0) - for mode in ("warmup", "capture", "replay", "fallback", "prepare_error"): - with self.subTest(mode=mode): - selected = None if mode == "fallback" else key - engine, runner, resource_manager, _, outputs = _make_decoder_runner(selected) - metadata = self._metadata() - engine.attn_metadata = metadata - engine._metadata.setup_attention.return_value = metadata - runner.maybe_get_cuda_graph.return_value = (metadata, None, selected) - runner.needs_capture.return_value = mode in ("warmup", "capture") - runner.is_warmup_only = mode == "warmup" - runner.capture.return_value = outputs - batch = ScheduledRequests() - batch.generation_requests = [_make_request_stub(1)] - prepared = engine._inputs.prepare_inputs.return_value - - def prepare(*args: object, **kwargs: object) -> PreparedInputs: - self.assertIs(metadata.trtllm_gen_jit_warmup, False) - self.assertEqual(metadata._csa2_defer_decode_outputs, mode != "fallback") - if mode == "prepare_error": - raise RuntimeError("prepare failed") - return prepared - - engine._inputs.prepare_inputs.side_effect = prepare - with patch( - "tensorrt_llm._torch.pyexecutor.engine.runners.decoder.inputs.torch.cuda.Event", - return_value=Mock(), - ): - if mode == "prepare_error": - with self.assertRaisesRegex(RuntimeError, "prepare failed"): - engine.forward( - ScheduledInputs(batch=batch), resource_manager=resource_manager - ) - else: - actual_outputs = engine.forward( - ScheduledInputs(batch=batch), resource_manager=resource_manager - ) - self.assertEqual(actual_outputs, {**outputs, "runtime_draft_len": 0}) - self.assertIs(actual_outputs["logits"], outputs["logits"]) - self.assertFalse(metadata._csa2_defer_decode_outputs) - if mode in ("warmup", "capture"): - runner.capture.assert_called_once() - if mode in ("capture", "replay"): - runner.replay.assert_called_once() - if mode == "fallback": - engine._forward_step.assert_called_once() - def test_jit_warmup_flag_marks_only_the_warmup_forward(self) -> None: engine, _, resource_manager, _, _ = _make_decoder_runner(None) runner = engine diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py index 87ee1978acec..58af4d6b781d 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/test_warmup.py @@ -33,6 +33,7 @@ import tensorrt_llm._torch.pyexecutor.engine.model_call as model_call_module import tensorrt_llm._torch.pyexecutor.engine.runners.decoder.graphs as graphs_module import tensorrt_llm._torch.pyexecutor.engine.runners.decoder.warmup as decoder_warmup_module +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.custom_ops.torch_custom_ops import MXFP8GemmRunner from tensorrt_llm._torch.models.modeling_utils import remove_weights @@ -307,7 +308,10 @@ def test_warmup_builds_dsa_attn_metadata_when_no_forward_ran(self) -> None: """The DSA top-K pre-compile hooks read attn_metadata; when every warmup forward was skipped it is built on demand so the engines still compile before serving.""" - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAtrtllmAttentionMetadata + from tensorrt_llm._torch.attention.backends.sparse.dsa import ( + DSAMetadata, + DSAtrtllmAttentionMetadata, + ) engine = object.__new__(DecoderWarmup) engine._config = SimpleNamespace( @@ -323,10 +327,14 @@ def test_warmup_builds_dsa_attn_metadata_when_no_forward_ran(self) -> None: buffers=None, attention_runtime_features=None, ) - metadata = Mock(spec=DSAtrtllmAttentionMetadata) + dsa = Mock(spec=DSAMetadata) + metadata = Mock() + metadata.find_ext.side_effect = lambda cls: dsa if cls is DSAMetadata else None def build( - kv_cache_manager: KVCacheManagerV2, draft_kv_cache_manager: KVCacheManagerV2 | None + kv_cache_manager: KVCacheManagerV2, + draft_kv_cache_manager: KVCacheManagerV2 | None, + spec_decoding: SpecDecodingConfig | None, ) -> Mock: engine._metadata.attn_metadata = metadata return metadata @@ -339,9 +347,9 @@ def build( engine._ensure_dsa_attn_metadata_for_warmup(resource_manager) engine._warmup_cute_dsl_radix_topk() - engine._metadata.setup_attention.assert_called_once_with(kv_cache_manager, None) - metadata.warmup_cute_dsl_radix_topk.assert_called_once_with(3) - metadata.warmup_selfsampling_topk.assert_called_once_with(3, batch_sizes=[8]) + engine._metadata.setup_attention.assert_called_once_with(kv_cache_manager, None, None) + dsa.warmup_cute_dsl_radix_topk.assert_called_once_with(3) + dsa.warmup_selfsampling_topk.assert_called_once_with(3, batch_sizes=[8]) def test_warmup_does_not_build_metadata_for_non_dsa_backend(self) -> None: engine = object.__new__(DecoderWarmup) diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/utils.py b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/utils.py index 63791003ec62..2b4824eb17ad 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/decoder/utils.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/decoder/utils.py @@ -208,7 +208,8 @@ def decoder_warmup( warmup_timer=_WarmupTimer(rank=0), metrics={}, agreement=agreement if agreement is not None else WarmupAgreement(None, mapping), - metadata=Mock(), + # Attention metadata with no extensions attached. + metadata=Mock(attn_metadata=Mock(find_ext=Mock(return_value=None))), dummy_requests=dummy_requests, graphs=graphs, inputs=SimpleNamespace(guided_decoder=None), diff --git a/tests/unittest/_torch/pyexecutor/engine/runners/test_encoder.py b/tests/unittest/_torch/pyexecutor/engine/runners/test_encoder.py index 03bed81689a6..5b8358ccad22 100644 --- a/tests/unittest/_torch/pyexecutor/engine/runners/test_encoder.py +++ b/tests/unittest/_torch/pyexecutor/engine/runners/test_encoder.py @@ -211,9 +211,10 @@ def test_encoder_only_incomplete_graph_config_warns_and_stays_eager() -> None: def test_encoder_only_attention_metadata_uses_runner_cache_indirection() -> None: cache_indirection = object() + block_ids, kv_block_ids = object(), object() metadata = SimpleNamespace( - block_ids_per_seq=object(), - kv_block_ids_per_seq=object(), + block_ids_per_seq=block_ids, + kv_block_ids_per_seq=kv_block_ids, ) runner = object.__new__(EncoderRunner) runner._model = SimpleNamespace(model_config=object()) @@ -221,6 +222,7 @@ def test_encoder_only_attention_metadata_uses_runner_cache_indirection() -> None max_batch_size=4, max_num_tokens=16, max_beam_width=2, + max_seq_len=8, attention_backend=TrtllmAttention, ) runner._attention_runtime_features = AttentionRuntimeFeatures() @@ -238,8 +240,10 @@ def test_encoder_only_attention_metadata_uses_runner_cache_indirection() -> None assert actual is metadata assert build_metadata.call_args.kwargs["cache_indirection"] is cache_indirection - assert metadata.block_ids_per_seq is None - assert metadata.kv_block_ids_per_seq is None + assert build_metadata.call_args.kwargs["max_seq_len"] == 8 + # Metadata without a KV cache manager owns its block IDs; the runner leaves them. + assert metadata.block_ids_per_seq is block_ids + assert metadata.kv_block_ids_per_seq is kv_block_ids def test_encoder_decoder_attention_metadata_omits_decoder_cache_indirection() -> None: @@ -253,6 +257,7 @@ def test_encoder_decoder_attention_metadata_omits_decoder_cache_indirection() -> max_batch_size=4, max_num_tokens=16, max_beam_width=2, + max_seq_len=8, attention_backend=TrtllmAttention, ) runner._attention_runtime_features = AttentionRuntimeFeatures() @@ -418,7 +423,7 @@ class _TrtllmSubclassMetadata(TrtllmAttentionMetadata): def test_encoder_runner_forwards_eager_model_inputs_and_gathers_logits() -> None: - metadata = SimpleNamespace(on_update_kv_lens=Mock()) + metadata = Mock() model_forward = Mock(return_value={"logits": torch.arange(12).reshape(3, 4)}) runner = object.__new__(EncoderRunner) runner._model_caller = model_forward @@ -436,7 +441,8 @@ def test_encoder_runner_forwards_eager_model_inputs_and_gathers_logits() -> None ) torch.testing.assert_close(outputs["logits"], torch.arange(12).reshape(3, 4)) - metadata.on_update_kv_lens.assert_called_once_with() + # The metadata was prepared with the inputs; the forward step leaves it as is. + assert metadata.method_calls == [] model_forward.assert_called_once_with(**inputs, return_context_logits=True) diff --git a/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py b/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py index f6ccb731c0cf..766de1ab30b9 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py @@ -95,6 +95,13 @@ def test_a_default_msa_policy_arms_nothing(): metadata = _metadata(DEFAULT_RUNTIME_HOOKS, object.__new__(TrtllmAttentionMetadata)) metadata._arm_msa_policy(None) + assert metadata._model_metadata_config() == {} + + +def test_model_declared_extensions_and_settings_reach_creation(): + hooks = RuntimeHooks(metadata_config=(("pools", {}),)) + metadata = _metadata(hooks, object.__new__(TrtllmAttentionMetadata)) + assert metadata._model_metadata_config() == {"pools": {}} def test_an_msa_policy_requires_the_msa_backend_and_cache(): @@ -120,6 +127,7 @@ def test_an_msa_policy_reaches_the_metadata_and_the_cache(): metadata._arm_msa_policy(manager) - assert metadata.attn_metadata.msa_policy is policy + # The policy reaches the metadata as creation configuration (msa_policy). + assert metadata._model_metadata_config() == {"msa_policy": policy} assert manager.skip_unchanged_block_offsets is True assert manager.reuse_pool_views is False diff --git a/tests/unittest/_torch/pyexecutor/engine/test_metadata.py b/tests/unittest/_torch/pyexecutor/engine/test_metadata.py index 18f2639f0d40..9568f3694d45 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_metadata.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_metadata.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from functools import partial from types import SimpleNamespace from typing import Any from unittest.mock import Mock @@ -9,10 +8,17 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends import core as core_module from tensorrt_llm._torch.attention.backends import interface as interface_module from tensorrt_llm._torch.attention.backends import trtllm as trtllm_module -from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata +from tensorrt_llm._torch.attention.backends.features import helix as helix_module +from tensorrt_llm._torch.attention.backends.interface import ( + AttentionBackend, + AttentionRuntimeFeatures, + BatchInputs, +) +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention, TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import KVCacheLayerSpec, ModelConfig from tensorrt_llm._torch.pyexecutor.engine.metadata import ( @@ -24,15 +30,28 @@ pytestmark = pytest.mark.cpu_only -class _AttentionMetadata: +class _AttentionMetadata(TrtllmAttentionMetadata): def __init__(self, **kwargs: Any) -> None: self.__dict__.update(kwargs) -class _AttentionBackend: +class _AttentionBackend(AttentionBackend): Metadata = _AttentionMetadata +def _backend_creating_with(constructor: Mock) -> type[AttentionBackend]: + """A backend whose Metadata(...) calls ``constructor`` with the creation arguments.""" + + class _ConstructorMetadata(_AttentionMetadata): + def __new__(cls, **kwargs: Any) -> Any: + return constructor(**kwargs) + + class _ConstructorBackend(_AttentionBackend): + Metadata = _ConstructorMetadata + + return _ConstructorBackend + + def test_build_attention_metadata_resolves_model_derived_values() -> None: sparse_metadata_params = object() sparse_attention_config = SimpleNamespace( @@ -63,7 +82,7 @@ def test_build_attention_metadata_resolves_model_derived_values() -> None: ) assert metadata.max_num_heads_per_kv == 4 - assert metadata.enable_flash_mla + assert metadata.extension_types == (FlashMlaMetadata,) assert metadata.enable_context_mla_with_cached_kv assert metadata.sparse_metadata_params is sparse_metadata_params sparse_attention_config.to_sparse_metadata_params.assert_called_once_with( @@ -215,7 +234,7 @@ def test_build_attention_metadata_explicit_disable( max_batch_size=4, max_num_tokens=16, max_beam_width=1, - attention_backend=SimpleNamespace(Metadata=constructor), + attention_backend=_backend_creating_with(constructor), attention_runtime_features=AttentionRuntimeFeatures( cache_reuse=cache_reuse, chunked_prefill=chunked_prefill ), @@ -252,9 +271,10 @@ def test_build_attention_metadata_helix_mla_preparation_lifecycle( ), ) monkeypatch.setattr(torch.Tensor, "cuda", lambda self, **kwargs: self.clone()) - monkeypatch.setattr(interface_module, "maybe_pin_memory", lambda tensor: tensor) + monkeypatch.setattr(core_module, "maybe_pin_memory", lambda tensor: tensor) monkeypatch.setattr(trtllm_module, "maybe_pin_memory", lambda tensor: tensor) monkeypatch.setattr(trtllm_module, "prefer_pinned", lambda: False) + monkeypatch.setattr(helix_module, "prefer_pinned", lambda: False) monkeypatch.setattr(trtllm_module, "get_model_extra_attrs", lambda: object()) mapping = Mapping(world_size=2, rank=rank, cp_size=2, cp_config={"cp_type": CpType.HELIX}) manager = SimpleNamespace( @@ -264,6 +284,7 @@ def test_build_attention_metadata_helix_mla_preparation_lifecycle( max_blocks_per_seq=8, max_seq_len=128, host_kv_cache_block_offsets=torch.zeros((1, 3, 2, 8), dtype=torch.int32), + prepare_batch_tables=Mock(), copy_batch_block_offsets=Mock(), ) model_config = SimpleNamespace( @@ -282,13 +303,7 @@ def test_build_attention_metadata_helix_mla_preparation_lifecycle( max_batch_size=3, max_num_tokens=32, max_beam_width=1, - attention_backend=SimpleNamespace( - Metadata=partial( - TrtllmAttentionMetadata, - workspace=torch.empty(0), - cuda_graph_workspace=torch.empty(0), - ) - ), + attention_backend=TrtllmAttention, attention_runtime_features=AttentionRuntimeFeatures(), mapping=mapping, cache_indirection=None, @@ -301,13 +316,16 @@ def test_build_attention_metadata_helix_mla_preparation_lifecycle( ([1], 0, [10]), ([3, 19, 1], 2, [0, 0, 11]), ]: - metadata.num_contexts = num_contexts - metadata.seq_lens = torch.tensor(lengths, dtype=torch.int32) - metadata.prompt_lens = lengths - metadata.request_ids = list(range(len(lengths))) - metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached) metadata.update_helix_param([cached[-1]], [False]) - metadata.prepare() + metadata.prepare( + BatchInputs( + seq_lens=torch.tensor(lengths, dtype=torch.int32), + num_contexts=num_contexts, + request_ids=list(range(len(lengths))), + prompt_lens=lengths, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached), + ) + ) assert metadata.num_ctx_cached_tokens == 0 assert metadata.max_ctx_cached_token_len == 0 @@ -345,7 +363,7 @@ def test_build_attention_metadata_dense_sparse_config_rejects_explicit_disable() max_batch_size=4, max_num_tokens=16, max_beam_width=1, - attention_backend=SimpleNamespace(Metadata=constructor), + attention_backend=_backend_creating_with(constructor), attention_runtime_features=AttentionRuntimeFeatures(), mapping=Mapping(world_size=2, cp_size=2, cp_config={"cp_type": CpType.HELIX}), cache_indirection=None, diff --git a/tests/unittest/_torch/pyexecutor/engine_test_utils.py b/tests/unittest/_torch/pyexecutor/engine_test_utils.py index 0aa5b374e05e..f7bb3788aebb 100644 --- a/tests/unittest/_torch/pyexecutor/engine_test_utils.py +++ b/tests/unittest/_torch/pyexecutor/engine_test_utils.py @@ -14,6 +14,8 @@ import tensorrt_llm import tensorrt_llm._torch.pyexecutor.engine.runners.decoder.warmup as decoder_warmup_module +from tensorrt_llm._torch.attention.backends.interface import AttentionBackend +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.pyexecutor.engine.runners.decoder import DecoderRunner from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.warmup import WarmupAgreement @@ -278,15 +280,18 @@ def _allgather(value: int) -> list[int]: return WarmupAgreement(dist, mapping) -class _AttentionMetadata: +class _AttentionMetadata(TrtllmAttentionMetadata): + """Records its creation arguments; like the TRTLLM core, it has block-ID + tables only with a KV cache manager.""" + def __init__(self, **kwargs: object) -> None: self.__dict__.update(kwargs) - self.block_ids_per_seq = object() - self.kv_block_ids_per_seq = object() - self.on_update_kv_lens = Mock() + has_cache = kwargs.get("kv_cache_manager") is not None + self.__dict__["block_ids_per_seq"] = object() if has_cache else None + self.__dict__["kv_block_ids_per_seq"] = object() if has_cache else None -class _AttentionBackend: +class _AttentionBackend(AttentionBackend): Metadata = _AttentionMetadata diff --git a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py index 9ddd9429d3b9..dfb3bd175deb 100644 --- a/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py +++ b/tests/unittest/_torch/pyexecutor/kv_cache/test_kvcm2_integration.py @@ -18,12 +18,14 @@ import sys from collections.abc import Callable from dataclasses import dataclass, field, replace +from functools import partial from types import SimpleNamespace from unittest.mock import ANY, Mock, call, patch import numpy as np import pytest import torch +from _torch.helpers import give_stub_a_draft from tensorrt_llm._torch.distributed.communicator import Distributed, ReduceOp from tensorrt_llm._torch.pyexecutor._util import _get_num_storage_pool_groups @@ -2677,6 +2679,14 @@ def _disagg_draft_fixture( return draft, occupant, request +def _stub_set_seq_lens(metadata, rows, *, host=None, device=None, update_counts=False): + """``set_seq_lens`` of a stub metadata: writes the host and device query lengths of ``rows``.""" + if host is not None: + metadata._seq_lens[rows] = host + if device is not None: + metadata._seq_lens_cuda[rows] = device + + def _disagg_generation_step(draft: KVCacheManagerV2, request: LlmRequest) -> None: """One executor generation step on the draft manager: grow the mirror, then the post-forward update_resources (nothing accepted, one token).""" @@ -2772,13 +2782,14 @@ def test_disagg_generation_drafter_writes_stay_on_its_own_draft_pages( worker.set_draft_model(draft_model) attn_metadata = SimpleNamespace( max_seq_len=1 << 17, - draft_kv_cache_block_offsets=offsets, num_contexts=0, num_seqs=1, num_ctx_tokens=0, _seq_lens_cuda=torch.zeros(1, dtype=torch.int32, device="cuda"), _seq_lens=torch.zeros(1, dtype=torch.int32), ) + attn_metadata.set_seq_lens = partial(_stub_set_seq_lens, attn_metadata) + give_stub_a_draft(attn_metadata, offsets) worker._lazy_init_ctx_buffers( draft_model, SimpleNamespace(max_num_requests=2), @@ -2923,7 +2934,7 @@ def test_slot_from_zero_worker_refuses_a_disagg_windowed_draft_pool( offsets = torch.zeros( (num_pools, 2, 2, draft.max_blocks_per_seq), dtype=torch.int32, device="cuda" ) - attn_metadata = SimpleNamespace(max_seq_len=1 << 17, draft_kv_cache_block_offsets=offsets) + attn_metadata = give_stub_a_draft(SimpleNamespace(max_seq_len=1 << 17), offsets) def bind() -> None: """Bind the drafter's ctx cache to the draft pool.""" @@ -2994,13 +3005,14 @@ def test_trailing_drafter_slot_reads_only_pages_its_request_holds( worker.set_draft_model(draft_model) attn_metadata = SimpleNamespace( max_seq_len=1 << 17, - draft_kv_cache_block_offsets=offsets, num_contexts=0, num_seqs=1, num_ctx_tokens=0, _seq_lens_cuda=torch.zeros(1, dtype=torch.int32, device="cuda"), _seq_lens=torch.zeros(1, dtype=torch.int32), ) + attn_metadata.set_seq_lens = partial(_stub_set_seq_lens, attn_metadata) + give_stub_a_draft(attn_metadata, offsets) worker._lazy_init_ctx_buffers( draft_model, SimpleNamespace(max_num_requests=2), diff --git a/tests/unittest/_torch/pyexecutor/test_context_logits_memory.py b/tests/unittest/_torch/pyexecutor/test_context_logits_memory.py index c99ff99d97e1..0049cb8b6eda 100644 --- a/tests/unittest/_torch/pyexecutor/test_context_logits_memory.py +++ b/tests/unittest/_torch/pyexecutor/test_context_logits_memory.py @@ -366,7 +366,6 @@ def run(graph_key, inputs, run_model, postprocess_inputs, **kwargs): _metadata=SimpleNamespace(setup_attention=lambda *args: metadata, attn_metadata=metadata), _moe_load_balancer=None, _stage_in_graph_sampling=None, - _cuda_graph_metadata_prepare_scope=lambda *args, **kwargs: nullcontext(), _start_beam_histories=lambda batch: None, _maybe_bypass_torch_compile=bypass, _forward_step=forward, diff --git a/tests/unittest/_torch/pyexecutor/test_cuda_graph_capture_replay.py b/tests/unittest/_torch/pyexecutor/test_cuda_graph_capture_replay.py index 8f4e4010db1b..08a55349632d 100644 --- a/tests/unittest/_torch/pyexecutor/test_cuda_graph_capture_replay.py +++ b/tests/unittest/_torch/pyexecutor/test_cuda_graph_capture_replay.py @@ -24,6 +24,9 @@ TLLM_CUDA_GRAPH_STRICT_BUFFERS checks their data_ptr() stays stable across replay() calls, catching a rebind (vs. an in-place update) to stale memory. +TestPrepareEachForward guards that warmup and capture never forward the +attention metadata twice for one prepare when the caller asks for it. + TestCaptureAllowedGate guards allow_capture(): live capture must stay confined to warmup, since capturing outside it could resize shared buffers and invalidate addresses already baked into other graphs. @@ -35,20 +38,60 @@ import torch from _torch.helpers import create_mock_cuda_graph_runner -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor import cuda_graph_runner as cuda_graph_runner_module from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import KeyType from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests -from tensorrt_llm._torch.speculative.interface import ( - prepare_attn_metadata_for_draft_replay, - restore_attn_metadata_after_draft_replay, -) pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") SENTINEL = -12345 +class _PrepareCountingMetadata: + """Attention metadata that records the prepares and forwards it sees.""" + + def __init__(self): + self.context_requests = ["request"] + self.prefill_cuda_graphs = True + self.events = [] + + def prepare(self, inputs=None, *, context_requests=None, prefill_cuda_graphs=False): + assert inputs is None + assert context_requests == self.context_requests + assert prefill_cuda_graphs is self.prefill_cuda_graphs + self.events.append("prepare") + + +class TestPrepareEachForward: + def _capture(self, prepare_each_forward): + runner = create_mock_cuda_graph_runner(1) + key = KeyType(batch_size=1, draft_len=0) + num_tokens = runner._get_num_tokens_for_key(key) + attn_metadata = _PrepareCountingMetadata() + + def forward_fn(inputs): + attn_metadata.events.append("forward") + return inputs["input_ids"].clone() + + inputs = { + "attn_metadata": attn_metadata, + "input_ids": torch.zeros((num_tokens,), device="cuda", dtype=torch.int32), + "position_ids": torch.zeros((1, num_tokens), device="cuda", dtype=torch.int32), + } + runner.capture(key, forward_fn, inputs, prepare_each_forward=prepare_each_forward) + return runner, attn_metadata.events + + def test_every_forward_after_the_first_follows_its_own_prepare(self): + runner, events = self._capture(prepare_each_forward=True) + # The caller prepared the first warmup forward. + expected = ["forward"] + ["prepare", "forward"] * runner.WARMUP_STEPS + assert events == expected + + def test_no_prepare_without_the_request(self): + runner, events = self._capture(prepare_each_forward=False) + assert events == ["forward"] * (runner.WARMUP_STEPS + 1) + + class TestCaptureReplayStaticTensors: """Invariants CUDAGraphRunner.capture()/replay() must uphold for shared_static_tensors: every key present at capture time is fully @@ -418,27 +461,6 @@ def __init__(self, value: int): self.some_buf = torch.full((1,), value, device="cuda", dtype=torch.int32) -class _DraftReplayMetadataStub(TrtllmAttentionMetadata): - """attn_metadata with only the fields the draft-replay helpers swap; skips __init__.""" - - def __init__(self, value: int): - self.draft_replay_swapped_attrs = {} - self.kv_cache_manager = object() - self.kv_cache_block_offsets = torch.full((1,), value, device="cuda", dtype=torch.int32) - self.host_kv_cache_block_offsets = torch.zeros(1, dtype=torch.int32) - self.draft_kv_cache_block_offsets = torch.full( - (1,), value + 1000, device="cuda", dtype=torch.int32 - ) - self.some_alias = self.kv_cache_block_offsets - self.enable_flash_mla = False - - def prepare_for_draft_forward(self): - # Mirrors DSA's _fullkv aliases, rebound as a side effect of the swap. - self.record_draft_swap("some_alias") - self.some_alias = self.kv_cache_block_offsets - return None - - class TestStrictBufferCheck: """Graph-visible metadata tensors must keep their data_ptr() from capture to replay.""" @@ -513,38 +535,6 @@ def rebind(fn_inputs): assert len(calls) == runner.WARMUP_STEPS assert runner.replay(key, inputs).item() == 99 - def test_replay_accepts_swapped_attr_with_unchanged_saved_target(self, monkeypatch): - """A swapped attr is validated via its saved target; clearing the swap flags it.""" - runner, key, attn_metadata, inputs = self._make_runner_and_inputs(monkeypatch, value=10) - runner.capture(key, self._forward_reading_some_buf, inputs) - - # Keep the captured tensor alive so the graph doesn't read freed memory. - captured_buf = attn_metadata.some_buf - attn_metadata.some_buf = torch.full((1,), 99, device="cuda", dtype=torch.int32) - - attn_metadata.draft_replay_swapped_attrs = {"some_buf": captured_buf} - # The graph still reads the captured buffer (10), not the rebound one (99). - assert runner.replay(key, inputs).item() == 10 - - attn_metadata.draft_replay_swapped_attrs = {} - with pytest.raises(RuntimeError, match="some_buf"): - runner.replay(key, inputs) - del captured_buf - - def test_draft_replay_swapped_attrs_does_not_mask_other_rebinds(self, monkeypatch): - """Exempting one swapped attr still flags a rebind of another attr.""" - runner, key, attn_metadata, inputs = self._make_runner_and_inputs(monkeypatch, value=10) - attn_metadata.other_buf = torch.full((1,), 1, device="cuda", dtype=torch.int32) - runner.capture(key, self._forward_reading_some_buf, inputs) - - captured_buf = attn_metadata.some_buf - attn_metadata.some_buf = torch.full((1,), 99, device="cuda", dtype=torch.int32) - attn_metadata.other_buf = torch.full((1,), 2, device="cuda", dtype=torch.int32) - - attn_metadata.draft_replay_swapped_attrs = {"some_buf": captured_buf} - with pytest.raises(RuntimeError, match="other_buf"): - runner.replay(key, inputs) - def test_replay_skips_graph_temporary(self, monkeypatch): """A rebound graph temporary is ignored; other rebinds are still flagged.""" runner, key, attn_metadata, inputs = self._make_runner_and_inputs(monkeypatch, value=10) @@ -566,80 +556,6 @@ def forward_fn(fn_inputs): runner.replay(key, inputs) del captured_tmp_buf - def _make_draft_replay_runner_and_inputs(self, monkeypatch, value): - runner, key, _, inputs = self._make_runner_and_inputs(monkeypatch, value) - attn_metadata = _DraftReplayMetadataStub(value) - inputs["attn_metadata"] = attn_metadata - draft_kv_cache_manager = SimpleNamespace( - host_kv_cache_block_offsets=torch.zeros(1, dtype=torch.int32) - ) - return runner, key, attn_metadata, inputs, draft_kv_cache_manager - - @staticmethod - def _forward_reading_block_offsets(fn_inputs): - return fn_inputs["input_ids"].clone() + fn_inputs["attn_metadata"].kv_cache_block_offsets - - @pytest.mark.parametrize("attr", ["kv_cache_block_offsets", "some_alias"]) - @pytest.mark.parametrize("replace_target", [False, True]) - def test_draft_swap_lifecycle(self, monkeypatch, attr, replace_target): - """A valid draft swap replays; a target replaced after capture is flagged.""" - runner, key, attn_metadata, inputs, draft_mgr = self._make_draft_replay_runner_and_inputs( - monkeypatch, value=10 - ) - runner.capture(key, self._forward_reading_block_offsets, inputs) - - # Keep the captured tensor alive so the graph doesn't read freed memory. - captured_buf = attn_metadata.kv_cache_block_offsets - if replace_target: - setattr(attn_metadata, attr, torch.full((1,), 99, device="cuda", dtype=torch.int32)) - - saved = prepare_attn_metadata_for_draft_replay(attn_metadata, draft_mgr) - try: - if replace_target: - with pytest.raises(RuntimeError, match=attr): - runner.replay(key, inputs) - else: - assert runner.replay(key, inputs).item() == 10 - finally: - restore_attn_metadata_after_draft_replay(attn_metadata, saved) - del captured_buf - - def test_restore_clears_swapped_attrs_and_target_buffer(self, monkeypatch): - """Restore leaves no swapped attrs behind and rebinds the target buffer.""" - _, _, attn_metadata, _, draft_mgr = self._make_draft_replay_runner_and_inputs( - monkeypatch, value=10 - ) - target_buf = attn_metadata.kv_cache_block_offsets - - saved = prepare_attn_metadata_for_draft_replay(attn_metadata, draft_mgr) - assert attn_metadata.draft_replay_swapped_attrs["kv_cache_block_offsets"] is target_buf - assert attn_metadata.draft_replay_swapped_attrs["some_alias"] is target_buf - assert attn_metadata.kv_cache_block_offsets is attn_metadata.draft_kv_cache_block_offsets - - # A repeated swap keeps the first recorded original. - attn_metadata.swap_for_draft("kv_cache_block_offsets", torch.zeros(1, device="cuda")) - assert attn_metadata.draft_replay_swapped_attrs["kv_cache_block_offsets"] is target_buf - - restore_attn_metadata_after_draft_replay(attn_metadata, saved) - - assert attn_metadata.draft_replay_swapped_attrs == {} - assert attn_metadata.kv_cache_block_offsets is target_buf - assert attn_metadata.some_alias is target_buf - - def test_cuda_graph_metadata_copy_has_own_swap_record(self): - """A swap restored on a graph copy does not leak into the eager metadata's restore.""" - eager = TrtllmAttentionMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=None) - graph = eager.create_cuda_graph_metadata(1) - assert graph.draft_replay_swapped_attrs is not eager.draft_replay_swapped_attrs - - eager_buf = eager.kv_lens_cuda - graph.swap_for_draft("kv_lens_cuda", torch.zeros_like(graph.kv_lens_cuda)) - graph.restore_draft_swaps() - eager.swap_for_draft("kv_lens_cuda", torch.zeros_like(eager_buf)) - eager.restore_draft_swaps() - - assert eager.kv_lens_cuda is eager_buf - class TestStrictBufferCheckEnvVar: """TLLM_CUDA_GRAPH_STRICT_BUFFERS parsing; the tests above patch the flag directly.""" diff --git a/tests/unittest/_torch/pyexecutor/test_cuda_graph_runner.py b/tests/unittest/_torch/pyexecutor/test_cuda_graph_runner.py index b593ab930904..8bebced9fb19 100644 --- a/tests/unittest/_torch/pyexecutor/test_cuda_graph_runner.py +++ b/tests/unittest/_torch/pyexecutor/test_cuda_graph_runner.py @@ -23,8 +23,6 @@ EncoderCUDAGraphRunnerConfig, KeyType, SampleType, - _restore_spec_decode_capture_state, - _save_spec_decode_capture_state, ) from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests @@ -293,25 +291,6 @@ def test_graph_lookup_forwards_promoted_context_ids(self) -> None: self.assertEqual(result, (graph_attn_metadata, graph_spec_metadata, key)) -class CaptureStateTestCase(unittest.TestCase): - def test_spec_decode_capture_restores_kv_lens_between_warmups(self) -> None: - attn_metadata = Mock() - attn_metadata.num_seqs = 1 - attn_metadata.kv_lens_cuda = torch.tensor([4095], dtype=torch.int32) - - saved_kv_lens_cuda = _save_spec_decode_capture_state(attn_metadata, enable_spec_decode=True) - - # CUDA graph capture performs two eager warmup forwards. A speculative - # draft loop may advance the static attention metadata during each - # forward, but the next warmup must start from the original input. - for _ in range(2): - attn_metadata.kv_lens_cuda.add_(1) - _restore_spec_decode_capture_state(attn_metadata, saved_kv_lens_cuda) - self.assertEqual(attn_metadata.kv_lens_cuda.tolist(), [4095]) - - self.assertEqual(attn_metadata.on_update_kv_lens.call_count, 2) - - def test_encoder_capture_discards_warmup_and_capture_outputs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py b/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py index 3708b3b2a082..b910d143be76 100644 --- a/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py +++ b/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py @@ -1,8 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""DecoderInputPreparer.preprocess_inputs re-derives the KV-length metadata once, -after the overlap scheduler's KV-length correction, on exactly the steps that correct it, and leaves the inputs as -before.""" +"""DecoderInputPreparer.preprocess_inputs corrects only the generation positions. + +The overlap scheduler's KV-length correction reaches the attention metadata as +BatchInputs.kv_lens_offsets and is applied in its prepare, so preprocess_inputs +neither changes the device KV lengths nor calls the metadata. +""" import types @@ -12,11 +15,15 @@ from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.inputs import DecoderInputPreparer from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks +pytestmark = pytest.mark.cpu_only + class _Metadata: - """The fields preprocess_inputs reads; on_update_kv_lens records the kv_lens it saw.""" + """The fields preprocess_inputs reads; any method call is recorded.""" - def __init__(self, gen: int, tokens_per_gen: int, with_kv_cache: bool = True): + supports_kv_lens_offsets = True + + def __init__(self, gen: int, with_kv_cache: bool = True): self.kv_cache_manager = object() if with_kv_cache else None self.num_seqs = gen self.num_contexts = 0 @@ -24,49 +31,51 @@ def __init__(self, gen: int, tokens_per_gen: int, with_kv_cache: bool = True): self.num_ctx_tokens = 0 self.num_chunked_ctx_requests = 0 self.kv_lens_cuda = torch.arange(100, 100 + gen, dtype=torch.int32) - self.seen: list[list[int]] = [] + self.calls: list[str] = [] + + def __getattr__(self, name: str): + def record(*args, **kwargs) -> None: + self.calls.append(name) - def on_update_kv_lens(self) -> None: - self.seen.append(self.kv_lens_cuda.tolist()) + return record -def _preparer(once: bool, disable_overlap: bool = False) -> DecoderInputPreparer: - r = object.__new__(DecoderInputPreparer) - r.runtime_hooks = RuntimeHooks(kv_lens_update_once=once) - r._config = types.SimpleNamespace(disable_overlap_scheduler=disable_overlap) - r._buffers = types.SimpleNamespace( +def _preparer(hooks: RuntimeHooks, disable_overlap: bool = False) -> DecoderInputPreparer: + preparer = object.__new__(DecoderInputPreparer) + preparer.runtime_hooks = hooks + preparer._config = types.SimpleNamespace(disable_overlap_scheduler=disable_overlap) + preparer._buffers = types.SimpleNamespace( previous_pos_id_offsets_cuda=torch.full((64,), -2, dtype=torch.int32), previous_kv_lens_offsets_cuda=torch.full((16,), -3, dtype=torch.int32), ) - r.mapping = types.SimpleNamespace(has_cp_helix=lambda: False) - r.decode_prologue = None - r.guided_decoder = None - return r + preparer.mapping = types.SimpleNamespace(has_cp_helix=lambda: False) + preparer.decode_prologue = None + preparer.guided_decoder = None + return preparer -def _inputs(md: _Metadata, tokens: int) -> dict: +def _inputs(metadata: _Metadata, tokens: int) -> dict: return { - "attn_metadata": md, + "attn_metadata": metadata, "input_ids": torch.zeros(tokens, dtype=torch.int32), "position_ids": torch.arange(tokens, dtype=torch.int32).view(1, -1), } @pytest.mark.parametrize("gen", [1, 4]) -def test_corrected_step_updates_once_with_the_same_inputs(gen) -> None: - t = 4 - outs = {} - for once in (False, True): - md = _Metadata(gen, t) - inputs = _preparer(once).preprocess_inputs( - _inputs(md, gen * t), enable_spec_decode=True, runtime_draft_len=t - 1 - ) - outs[once] = (md.seen, inputs["position_ids"].tolist(), md.kv_lens_cuda.tolist()) - corrected = [100 + i - 3 for i in range(gen)] - # The upstream path updates after correction for both legacy hook values. - assert outs[False][0] == [corrected] - assert outs[True][0] == [corrected] - assert outs[True][1:] == outs[False][1:] +def test_overlap_step_corrects_positions_only(gen) -> None: + hooks = RuntimeHooks() + tokens_per_gen = 4 + metadata = _Metadata(gen) + inputs = _preparer(hooks).preprocess_inputs( + _inputs(metadata, gen * tokens_per_gen), + enable_spec_decode=True, + runtime_draft_len=tokens_per_gen - 1, + ) + num_tokens = gen * tokens_per_gen + assert inputs["position_ids"].tolist() == [[i - 2 for i in range(num_tokens)]] + assert metadata.kv_lens_cuda.tolist() == [100 + i for i in range(gen)] + assert metadata.calls == [] @pytest.mark.parametrize( @@ -74,19 +83,17 @@ def test_corrected_step_updates_once_with_the_same_inputs(gen) -> None: [(False, False, True), (True, True, True), (True, False, False)], ids=["no spec decode", "no overlap scheduler", "no kv cache manager"], ) -def test_uncorrected_steps_keep_their_update(spec, disable_overlap, with_kv_cache) -> None: - """Steps that do not correct the lengths still re-derive them once (the per-forward invalidation), with the same - inputs as without the hook.""" - outs = {} - for once in (False, True): - md = _Metadata(2, 4, with_kv_cache) - inputs = _preparer(once, disable_overlap).preprocess_inputs( - _inputs(md, 8), enable_spec_decode=spec, runtime_draft_len=3 - ) - outs[once] = (md.seen, inputs["position_ids"].tolist(), md.kv_lens_cuda.tolist()) - assert len(outs[True][0]) == 1 - assert outs[True] == outs[False] - - -def test_off_by_default() -> None: - assert RuntimeHooks().kv_lens_update_once is False +def test_uncorrected_steps_leave_the_inputs(spec, disable_overlap, with_kv_cache) -> None: + metadata = _Metadata(2, with_kv_cache) + inputs = _preparer(RuntimeHooks(), disable_overlap).preprocess_inputs( + _inputs(metadata, 8), enable_spec_decode=spec, runtime_draft_len=3 + ) + assert inputs["position_ids"].tolist() == [list(range(8))] + assert metadata.kv_lens_cuda.tolist() == [100, 101] + assert metadata.calls == [] + + +def test_the_retired_step_correction_hooks_are_gone() -> None: + hooks = RuntimeHooks() + assert not hasattr(hooks, "kv_lens_update_once") + assert not hasattr(hooks, "spec_offsets_op") diff --git a/tests/unittest/_torch/pyexecutor/test_preprocess_inputs_kv_lens_hook.py b/tests/unittest/_torch/pyexecutor/test_preprocess_inputs_kv_lens_hook.py deleted file mode 100644 index 964725a6a423..000000000000 --- a/tests/unittest/_torch/pyexecutor/test_preprocess_inputs_kv_lens_hook.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""preprocess_inputs calls on_update_kv_lens once per step. - -Under the overlap scheduler with speculative decoding the runner corrects -kv_lens_cuda and calls the hook after the correction; the call before it is -dropped. Without a correction the single call before it is kept. -""" - -from types import SimpleNamespace - -import pytest -import torch - -from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.inputs import DecoderInputPreparer - -pytestmark = pytest.mark.cpu_only - -KV_LENS = torch.tensor([10, 20, 30], dtype=torch.int32) -KV_LENS_OFFSETS = torch.tensor([1, 2, 0], dtype=torch.int32) -TOKENS_PER_REQUEST = 3 -NUM_TOKENS = KV_LENS.numel() * TOKENS_PER_REQUEST - - -class _Metadata: - """The slice of TrtllmAttentionMetadata that preprocess_inputs touches.""" - - def __init__(self): - self.kv_cache_manager = object() - self.num_seqs = KV_LENS.numel() - self.num_contexts = 0 - self.num_generations = KV_LENS.numel() - self.num_ctx_tokens = 0 - self.num_chunked_ctx_requests = 0 - self.kv_lens_cuda = KV_LENS.clone() - # kv_lens_cuda as seen by each on_update_kv_lens() call. - self.kv_lens_at_hook = [] - - def on_update_kv_lens(self): - self.kv_lens_at_hook.append(self.kv_lens_cuda.clone()) - - -def _run_preprocess_inputs(metadata, *, spec_decode, overlap): - engine = object.__new__(DecoderInputPreparer) - engine._config = SimpleNamespace(disable_overlap_scheduler=not overlap) - engine._buffers = SimpleNamespace( - previous_pos_id_offsets_cuda=torch.arange(NUM_TOKENS, dtype=torch.int32), - previous_kv_lens_offsets_cuda=KV_LENS_OFFSETS.clone(), - ) - engine.mapping = SimpleNamespace(has_cp_helix=lambda: False) - engine.guided_decoder = None - inputs = { - "input_ids": torch.zeros(NUM_TOKENS, dtype=torch.int32), - "position_ids": torch.zeros(1, NUM_TOKENS, dtype=torch.int32), - "attn_metadata": metadata, - } - engine.preprocess_inputs(inputs, enable_spec_decode=spec_decode, runtime_draft_len=0) - - -@pytest.mark.parametrize( - "spec_decode,overlap,corrected", - [(True, True, True), (True, False, False), (False, True, False)], -) -def test_hook_runs_once_per_step(spec_decode, overlap, corrected): - metadata = _Metadata() - _run_preprocess_inputs(metadata, spec_decode=spec_decode, overlap=overlap) - expected = KV_LENS + KV_LENS_OFFSETS if corrected else KV_LENS - assert len(metadata.kv_lens_at_hook) == 1 - assert torch.equal(metadata.kv_lens_at_hook[0], expected) diff --git a/tests/unittest/_torch/pyexecutor/test_runtime_hooks.py b/tests/unittest/_torch/pyexecutor/test_runtime_hooks.py index 9cd4bac088d3..90e8f2f19a61 100644 --- a/tests/unittest/_torch/pyexecutor/test_runtime_hooks.py +++ b/tests/unittest/_torch/pyexecutor/test_runtime_hooks.py @@ -72,7 +72,8 @@ def test_every_field_defaults_to_off(): getattr(value, item.name) in (False, None) for item in dataclasses.fields(value) ) else: - assert value in (False, None), field.name + # Tuple fields (model metadata extensions and their settings) default to empty. + assert value in (False, None, ()), field.name def test_hooks_declared_as_none_mean_the_default(): @@ -563,11 +564,16 @@ def test_the_decode_prologue_call_site_uses_its_protocol(): class _AttnMetadata: + """Attention metadata that records every method call; preprocess_inputs makes none.""" + def __init__(self) -> None: - self.kv_lens_updates = 0 + self.calls: list[str] = [] + + def __getattr__(self, name: str): + def record(*args, **kwargs) -> None: + self.calls.append(name) - def on_update_kv_lens(self) -> None: - self.kv_lens_updates += 1 + return record def _prologue_preparer(prologue): @@ -591,7 +597,7 @@ def test_preprocess_inputs_runs_the_native_prologue_without_a_hook(): assert preparer.preprocess_inputs(inputs, enable_spec_decode=False, runtime_draft_len=0) is ( inputs ) - assert inputs["attn_metadata"].kv_lens_updates == 1 + assert inputs["attn_metadata"].calls == [] @pytest.mark.parametrize("engaged", [False, True], ids=["declined", "engaged"]) @@ -609,7 +615,7 @@ def prologue(preparer, inputs, *, enable_spec_decode, runtime_draft_len): inputs ) assert calls == [(preparer, inputs, False, 3)] - assert inputs["attn_metadata"].kv_lens_updates == (0 if engaged else 1) + assert inputs["attn_metadata"].calls == [] # -- the execution-stream loop -------------------------------------------------------------------- diff --git a/tests/unittest/_torch/pyexecutor/test_steady_spec_prepare.py b/tests/unittest/_torch/pyexecutor/test_steady_spec_prepare.py index e771351d972b..d72b4d63a898 100644 --- a/tests/unittest/_torch/pyexecutor/test_steady_spec_prepare.py +++ b/tests/unittest/_torch/pyexecutor/test_steady_spec_prepare.py @@ -14,7 +14,10 @@ from _torch.pyexecutor.engine_test_utils import DummyModel import tensorrt_llm -from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures +from tensorrt_llm._torch.attention.backends.interface import ( + AttentionMetadata, + AttentionRuntimeFeatures, +) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import KeyType from tensorrt_llm._torch.pyexecutor.engine.runners.decoder.inputs import DecoderInputBuffers @@ -267,6 +270,8 @@ def replay(value, staged): # Plain values and tensors of a metadata object (other objects are shared or derived). _PLAIN = (int, float, bool, str, bytes, list, tuple, type(None), torch.Tensor) +# Records which prepare ran the step (set by the steady step only), not what the step hands the model. +_PATH_MARKERS = ("steady_decode_kv_lens",) def _snapshot(value): @@ -305,13 +310,12 @@ def _step_state(preparer, inputs, attn_metadata, spec_metadata, b, t): "previous_kv_lens_offsets": buffers.previous_kv_lens_offsets_cuda[:b], "gather_ids": buffers.gather_ids_cuda[:n], } - for prefix, obj in ( - ("attn", attn_metadata), - ("kv_cache_params", attn_metadata.kv_cache_params), - ): - for name, value in vars(obj).items(): - if isinstance(value, _PLAIN): - state[f"{prefix}.{name}"] = value + for name, value in _metadata_fields(attn_metadata, _PLAIN).items(): + if name.rsplit(".", 1)[-1] not in _PATH_MARKERS: + state[f"attn.{name}"] = value + for name, value in vars(attn_metadata.kv_cache_params).items(): + if isinstance(value, _PLAIN): + state[f"kv_cache_params.{name}"] = value for name in ( "num_tokens", "num_generations", @@ -341,6 +345,23 @@ def _repage_last_block(manager, request, other): assert after[0][0] == before[0][0] and after != before +def _metadata_fields(metadata, kinds=(torch.Tensor,), prefix="", seen=None): + """Fields of every extension of ``metadata`` that are instances of ``kinds``, nested metadata + (the draft) included, keyed by path.""" + seen = set() if seen is None else seen + seen.add(id(metadata)) + fields = {} + for extension in metadata.extensions: + for name, value in vars(extension).items(): + path = f"{prefix}{type(extension).__name__}.{name}" + if isinstance(value, AttentionMetadata): + if id(value) not in seen: + fields.update(_metadata_fields(value, kinds, f"{path}.", seen)) + elif isinstance(value, kinds): + fields[path] = value + return fields + + def _previous_step_tensors(gen, t, num_slots=8): """The previous step's device sampler outputs, per sequence slot.""" @@ -410,7 +431,7 @@ def test_prepare_inputs_steady_steps_match_forced_full_passes(): .setup_attention(manager) .create_cuda_graph_metadata(len(requests), False, draft_len) ) - assert type(attn_metadata) is TrtllmAttentionMetadata + assert isinstance(attn_metadata, TrtllmAttentionMetadata) spec_metadata = Mock( _force_non_greedy_for_capture=False, context_prompt_lookahead_tokens=None, @@ -418,13 +439,12 @@ def test_prepare_inputs_steady_steps_match_forced_full_passes(): ) arms[arm] = (preparer, attn_metadata, spec_metadata) # Both arms start from the same metadata buffer contents. - steady_fields, full_fields = (vars(arms[arm][1]) for arm in ("steady", "full")) + steady_fields, full_fields = (_metadata_fields(arms[arm][1]) for arm in ("steady", "full")) for name, value in steady_fields.items(): twin = full_fields.get(name) - if isinstance(value, torch.Tensor) and isinstance(twin, torch.Tensor): - if value.data_ptr() != twin.data_ptr(): - value.zero_() - twin.zero_() + if twin is not None and value.data_ptr() != twin.data_ptr(): + value.zero_() + twin.zero_() # Accepted tokens per request and step. None: request 1 re-pages its last block (which holds # draft slots only) and no request advances, so that is the batch's only block-table change. diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_arena_sizing.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_arena_sizing.py index 7581011fe63b..7e7cbc94e91a 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_arena_sizing.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_arena_sizing.py @@ -65,9 +65,13 @@ def _metadata(*, declares_max_seq_len): manager, no ``max_seq_len`` of its own. """ metadata = MagicMock( - spec=["kv_cache_manager", "max_seq_len"] if declares_max_seq_len else ["kv_cache_manager"] + spec=["kv_cache_manager", "find", "max_seq_len"] + if declares_max_seq_len + else ["kv_cache_manager", "find"] ) metadata.kv_cache_manager = MagicMock(max_seq_len=SERVED) + # No draft KV block offsets: the drafter stays on its private arena. + metadata.find = lambda name, default=None: default if declares_max_seq_len: metadata.max_seq_len = SERVED return metadata diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_block_tables.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_block_tables.py index a6c61a3a5cda..8807b246dfc5 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_block_tables.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_ctx_block_tables.py @@ -111,9 +111,11 @@ def test_refresh_ctx_block_tables_preserves_layer_offsets_and_source( ) original = source.clone() other_source = torch.full_like(source, 123) + # The draft metadata over the draft KV cache stages its own block offsets. + draft = SimpleNamespace(kv_cache_block_offsets=other_source if unified else source) metadata = SimpleNamespace( kv_cache_block_offsets=source if unified else other_source, - draft_kv_cache_block_offsets=other_source if unified else source, + find=lambda name, default=None: draft if name == "draft_or_none" else default, ) def decode_reference( diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_flashinfer_metadata.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_flashinfer_metadata.py deleted file mode 100644 index 79357a1bf783..000000000000 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_flashinfer_metadata.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""DFlash on attention metadata that keeps no device-side KV lengths (FlashInfer).""" - -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import torch - -from tensorrt_llm._torch.speculative.dflash import DFlashWorker - -pytestmark = pytest.mark.cpu_only - - -@pytest.mark.parametrize("has_kv_lens", [True, False], ids=["trtllm", "flashinfer"]) -@pytest.mark.parametrize("cuda_graph", [True, False], ids=["cuda_graph", "eager"]) -def test_spec_dec_save_skips_kv_lens_the_metadata_lacks(has_kv_lens: bool, cuda_graph: bool): - """A CUDA-graph step saves `kv_lens_cuda` only where the metadata has it. - - TRTLLM attention metadata keeps the KV lengths on the device; FlashInfer's - does not, and asking it to save the field raised. - """ - saved = [] - metadata = SimpleNamespace(prepare_for_spec_dec=lambda *fields: saved.append(fields)) - if has_kv_lens: - metadata.kv_lens_cuda = torch.zeros(2, dtype=torch.int32) - with patch.object(torch.cuda, "is_current_stream_capturing", return_value=False): - DFlashWorker._prepare_attn_metadata_for_dflash( - SimpleNamespace(), metadata, SimpleNamespace(is_cuda_graph=cuda_graph) - ) - expected = ["_seq_lens", "_seq_lens_cuda"] - if cuda_graph and has_kv_lens: - expected.append("kv_lens_cuda") - assert saved == [tuple(expected)] diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_guided_rollback.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_guided_rollback.py index 7960f49ba0cd..ed927ee3a052 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_guided_rollback.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_guided_rollback.py @@ -106,18 +106,14 @@ def test_dflash_worker_rolls_back_after_verification(monkeypatch): ) for stub in ( "_lazy_init_ctx_buffers", - "_prepare_attn_metadata_for_dflash", - "_prepare_kv_for_draft_forward", "_store_prefill_context", - "write_context_onehot_draft_probs", - "_restore_attn_metadata_from_spec_dec", ): monkeypatch.setattr(worker, stub, _noop) def finish_drafting(*args, **kwargs): guided.calls.append(("draft_complete", None)) - monkeypatch.setattr(worker, "_apply_kv_rewind_after_draft", finish_drafting) + monkeypatch.setattr(worker, "write_context_onehot_draft_probs", finish_drafting) monkeypatch.setattr(worker, "prepare_1st_drafter_inputs", lambda *a, **k: {}) monkeypatch.setattr(worker, "_prepare_next_new_tokens", lambda *a, **k: next_new) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_worker.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_worker.py index 2f9f9a1d4976..23764ca99e6e 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_worker.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash_worker.py @@ -19,12 +19,13 @@ DFlash accuracy test in ``integration/defs/accuracy/test_llm_api_pytorch.py``. """ -from types import SimpleNamespace +from types import MethodType, SimpleNamespace import pytest import torch from tensorrt_llm._device_copy import DeviceCopyBuffer +from tensorrt_llm._torch.attention.backends.core import AttentionCoreMetadata from tensorrt_llm._torch.speculative.dflash import DFlashWorker from tensorrt_llm.llmapi import DFlashDecodingConfig from tensorrt_llm.mapping import Mapping @@ -71,7 +72,7 @@ def test_trtllm_backend_builds_private_paged_context_cache(monkeypatch): draft_model = _FakeDraftModel() worker.set_draft_model(draft_model) spec_metadata = SimpleNamespace(max_num_requests=2) - attn_metadata = SimpleNamespace(max_seq_len=64) + attn_metadata = SimpleNamespace(max_seq_len=64, find=lambda name, default=None: default) worker._lazy_init_ctx_buffers(draft_model, spec_metadata, attn_metadata) @@ -145,6 +146,7 @@ def precompute_context_kv(projected, positions): _seq_lens_cuda=torch.zeros(num_gens, dtype=torch.int32, device="cuda"), _seq_lens=torch.zeros(num_gens, dtype=torch.int32), ) + attn_metadata.set_seq_lens = MethodType(AttentionCoreMetadata.set_seq_lens, attn_metadata) accepted_tokens = torch.arange(num_gens * tokens_per_req, dtype=torch.long, device="cuda").view( num_gens, tokens_per_req ) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_model_engine_runtime.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_model_engine_runtime.py index 4450c805b208..5fb348054df6 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_model_engine_runtime.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_model_engine_runtime.py @@ -9,10 +9,8 @@ import torch from tensorrt_llm._torch import utils as torch_utils -from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import ( - DeepseekV4TrtllmAttentionMetadata, -) -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata import DeepseekV4Metadata +from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDAGraphRunner, DSparkCUDAGraphRunner from tensorrt_llm._torch.pyexecutor.engine import metadata as metadata_module from tensorrt_llm._torch.pyexecutor.engine.runners.decoder import DecoderRunner @@ -172,6 +170,11 @@ def _warmup_metadata(position_helper, mask_helper): ) +def _container(extension): + """An attention metadata whose DeepSeek-V4 extension is ``extension`` (None: not attached).""" + return SimpleNamespace(find_ext=lambda extension_type: extension) + + def test_ragged_compressor_warmup_primes_and_cleans_intermediate_shape(monkeypatch): position_calls = [] mask_calls = [] @@ -185,7 +188,7 @@ def record_mask(new_comp, cu_new_comp, compressed_mask, batch_size, compress_rat metadata = _warmup_metadata(record_positions, record_mask) engine = SimpleNamespace( _ragged_verification_enabled=True, - attn_metadata=metadata, + attn_metadata=_container(metadata), _config=SimpleNamespace(max_batch_size=8, cuda_graph_batch_sizes=[1, 4, 8]), ) sync_calls = [] @@ -214,7 +217,7 @@ def fail_positions(*args): metadata = _warmup_metadata(fail_positions, lambda *_args: None) engine = SimpleNamespace( _ragged_verification_enabled=True, - attn_metadata=metadata, + attn_metadata=_container(metadata), _config=SimpleNamespace(max_batch_size=4, cuda_graph_batch_sizes=[1, 4]), ) monkeypatch.setattr(torch.cuda, "synchronize", lambda: None) @@ -234,7 +237,7 @@ def fail_positions(*args): @pytest.mark.parametrize("fail_after_mask", [False, True]) def test_ragged_compressor_warmup_calls_current_metadata_helpers(monkeypatch, fail_after_mask): """Run the real five-argument mask and position helper bodies on CPU tensors.""" - actual_mask = DeepseekV4TrtllmAttentionMetadata._compute_compressed_mask + actual_mask = DeepseekV4Metadata._compute_compressed_mask def mask(new_comp, cu_new_comp, compressed_mask, batch_size, compress_ratios): actual_mask(new_comp, cu_new_comp, compressed_mask, batch_size, compress_ratios) @@ -247,14 +250,12 @@ def mask(new_comp, cu_new_comp, compressed_mask, batch_size, compress_ratios): if fail_after_mask: raise RuntimeError("mask cleanup probe") - metadata = _warmup_metadata( - DeepseekV4TrtllmAttentionMetadata._compute_gen_compressed_position_ids, mask - ) + metadata = _warmup_metadata(DeepseekV4Metadata._compute_gen_compressed_position_ids, mask) original_storage = metadata.batch_size_cuda.data_ptr() original_gen_storage = metadata.gen_position_params_cuda.data_ptr() engine = SimpleNamespace( _ragged_verification_enabled=True, - attn_metadata=metadata, + attn_metadata=_container(metadata), _config=SimpleNamespace(max_batch_size=8, cuda_graph_batch_sizes=[1, 4, 8]), ) monkeypatch.setattr(torch_utils, "is_piecewise_running", lambda: True) @@ -287,7 +288,7 @@ def test_ragged_compressor_warmup_requires_trimmed_tokens(): def test_ragged_compressor_warmup_rejects_missing_metadata_contract(): engine = SimpleNamespace( _ragged_verification_enabled=True, - attn_metadata=SimpleNamespace(), + attn_metadata=_container(None), ) with pytest.raises(RuntimeError, match="missing .*max_draft_tokens"): @@ -434,36 +435,30 @@ def _metadata_inputs(metadata_type): return model, kwargs -@pytest.mark.parametrize("explicit_false", [False, True]) -def test_metadata_helper_keeps_native_constructor_kwargs(explicit_false): - seen = [] +class _RecordingMetadata(TrtllmAttentionMetadata): + """TRTLLM-family metadata that records its creation arguments instead of building.""" + + def __init__(self, **kwargs): + self.__dict__["creation_kwargs"] = kwargs - class NativeMetadata: - def __init__(self, **kwargs): - seen.append(kwargs) - model, kwargs = _metadata_inputs(NativeMetadata) +@pytest.mark.parametrize("explicit_false", [False, True]) +def test_metadata_helper_keeps_native_constructor_kwargs(explicit_false): + model, kwargs = _metadata_inputs(_RecordingMetadata) if explicit_false: kwargs["enable_ragged_verification"] = False - metadata_module.build_attention_metadata(model, **kwargs) + metadata = metadata_module.build_attention_metadata(model, **kwargs) - assert "enable_ragged_verification" not in seen[0] - assert seen[0]["max_num_heads_per_kv"] == 4 + assert "enable_ragged_verification" not in metadata.creation_kwargs + assert metadata.creation_kwargs["max_num_heads_per_kv"] == 4 def test_metadata_helper_enables_only_supported_ragged_metadata(): - class RaggedMetadata(DSAtrtllmAttentionMetadata): - def __init__(self, **kwargs): - self.constructor_flag = kwargs.pop("enable_ragged_verification", False) - self.enable_ragged_verification = self.constructor_flag - self.ragged_verify_lens = [1, 2] - - model, kwargs = _metadata_inputs(RaggedMetadata) + model, kwargs = _metadata_inputs(_RecordingMetadata) result = metadata_module.build_attention_metadata( model, enable_ragged_verification=True, **kwargs ) - assert result.constructor_flag is True - result._validate_ragged_verification() + assert result.creation_kwargs["enable_ragged_verification"] is True kwargs["attention_backend"] = SimpleNamespace(Metadata=object) with pytest.raises(ValueError, match="DSA target metadata"): @@ -474,19 +469,25 @@ def __init__(self, **kwargs): def test_decoder_enables_ragged_metadata_during_construction(monkeypatch, enabled): seen = [] - class MetadataProbe(DSAtrtllmAttentionMetadata): + class MetadataProbe(_RecordingMetadata): def __init__(self, **kwargs): - self.enable_ragged_verification = kwargs.pop("enable_ragged_verification", False) - self.kv_cache_manager = kwargs["kv_cache_manager"] - self.ragged_verify_lens = None - seen.append(self.enable_ragged_verification) + super().__init__(**kwargs) + seen.append(kwargs.get("enable_ragged_verification", False)) + + @property + def kv_cache_manager(self): + return self.creation_kwargs["kv_cache_manager"] model, kwargs = _metadata_inputs(MetadataProbe) decoder = object.__new__(DecoderMetadata) decoder.attn_metadata = None decoder.model = SimpleNamespace(model_config=model) decoder._config = SimpleNamespace( - **kwargs, spec_config=SimpleNamespace(enable_confidence_scheduling=enabled) + **kwargs, + spec_config=SimpleNamespace( + enable_confidence_scheduling=enabled, + spec_dec_mode=SimpleNamespace(use_one_engine=lambda: False), + ), ) decoder.mapping = kwargs["mapping"] decoder._attention_runtime_features = kwargs["attention_runtime_features"] @@ -495,9 +496,6 @@ def __init__(self, **kwargs): kv = object() result = decoder.setup_attention(kv) - if enabled: - result.ragged_verify_lens = [1, 2] - result._validate_ragged_verification() assert seen == [enabled] assert decoder.setup_attention(kv) is result @@ -738,7 +736,12 @@ def gather(*args) -> bool: state.new_tokens = torch.arange(12, dtype=torch.int32).reshape(4, 3) state.new_tokens_lens = torch.tensor([2, 3, 4], dtype=torch.int32) state.next_draft_tokens = torch.arange(9, dtype=torch.int32).reshape(3, 3) - metadata = SimpleNamespace(is_cuda_graph=False, prepare=lambda: None) + batches = [] + metadata = SimpleNamespace( + is_cuda_graph=False, + prepare=lambda batch, **_kwargs: batches.append(batch), + find_ext=lambda _extension_type: None, + ) prepared = decoder.prepare_inputs( scheduled, @@ -762,8 +765,9 @@ def gather(*args) -> bool: 4 - previous_lengths[1], ] assert inputs["input_ids"].numel() == sum(lengths) - assert metadata.seq_lens.tolist() == lengths - assert metadata.kv_cache_params.num_extra_kv_tokens == 2 + (batch,) = batches + assert batch.seq_lens.tolist() == lengths + assert batch.kv_cache_params.num_extra_kv_tokens == 2 assert decoder._gather_ids_buffer.tensor[: sum(lengths)].tolist() == list(range(sum(lengths))) assert draft_len == 3 assert gather_calls == ( diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_pard_guided_rollback.py b/tests/unittest/_torch/speculative/hw_agnostic/test_pard_guided_rollback.py index 95dde0535cff..05661f5d3ecd 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_pard_guided_rollback.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_pard_guided_rollback.py @@ -100,18 +100,13 @@ def test_pard_worker_rolls_back_after_verification(monkeypatch): monkeypatch.setattr( worker, "sample_and_accept_draft_tokens", lambda *a, **k: (accepted, num_accepted) ) - for stub in ( - "_prepare_attn_metadata_for_pard", - "_prepare_kv_for_draft_forward", - "write_context_onehot_draft_probs", - "_restore_attn_metadata_from_spec_dec", - ): + for stub in ("_prepare_kv_for_draft_forward",): monkeypatch.setattr(worker, stub, _noop) def finish_drafting(*args, **kwargs): guided.calls.append(("draft_complete", None)) - monkeypatch.setattr(worker, "_apply_kv_rewind_after_draft", finish_drafting) + monkeypatch.setattr(worker, "write_context_onehot_draft_probs", finish_drafting) monkeypatch.setattr(worker, "prepare_1st_drafter_inputs", lambda *a, **k: {}) monkeypatch.setattr(worker, "_prepare_next_new_tokens", lambda *a, **k: next_new) diff --git a/tests/unittest/_torch/speculative/test_eagle3.py b/tests/unittest/_torch/speculative/test_eagle3.py index 46a01a17aabb..0315f7fa9d22 100644 --- a/tests/unittest/_torch/speculative/test_eagle3.py +++ b/tests/unittest/_torch/speculative/test_eagle3.py @@ -17,7 +17,6 @@ import os import tempfile import unittest -from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock @@ -30,8 +29,8 @@ from tensorrt_llm import LLM, SamplingParams from tensorrt_llm._device_copy import DeviceCopyBuffer -from tensorrt_llm._torch.attention.backends.sparse.dsa import ( - DSACacheManagerV2, DSAtrtllmAttentionMetadata) +from tensorrt_llm._torch.attention.backends.features.spec_decoding import \ + SpecDecodingConfig from tensorrt_llm._torch.attention.backends.trtllm import \ TrtllmAttentionMetadata from tensorrt_llm._torch.metadata import KVCacheParams @@ -54,79 +53,9 @@ KvCacheConfig, MoeConfig, MTPDecodingConfig) -def test_mtp_eagle_refreshes_dsa_metadata_before_draft_forward() -> None: - """Refresh DSA mappings after switching to the draft cache.""" - events = [] - state = {} - draft_manager = object.__new__(DSACacheManagerV2) - - class _Metadata(DSAtrtllmAttentionMetadata): - - def __init__(self): - self._seq_lens_cuda = torch.tensor([1], dtype=torch.int32) - self._num_ctx_tokens = 0 - - def set_in_mtp_draft_loop(self, active): - state["in_mtp_draft_loop"] = active - - def set_mtp_num_accepted(self, value): - state["mtp_num_accepted"] = value - - def set_skip_topk(self, value): - state["skip_topk"] = value - - def on_update_kv_lens(self): - events.append("refresh") - - metadata = _Metadata() - - @contextmanager - def draft_context(attn_metadata, manager): - assert attn_metadata is metadata - assert manager is draft_manager - events.append("switch") - yield metadata - - class _StopAfterFirstDraftForward(Exception): - pass - - def run_draft_forward(*args, **kwargs): - events.append("forward") - raise _StopAfterFirstDraftForward - - worker = SimpleNamespace( - is_mtp_eagle=True, - draft_kv_cache_context=draft_context, - _run_draft_forward=run_draft_forward, - ) - - from tensorrt_llm._torch.speculative.eagle3 import Eagle3OneModelWorker - - with pytest.raises(_StopAfterFirstDraftForward): - Eagle3OneModelWorker._forward_linear_draft_loop( - worker, - {"position_ids": torch.tensor([0], dtype=torch.int32)}, - metadata, - SimpleNamespace(runtime_draft_len=1), - draft_model=object(), - draft_kv_cache_manager=draft_manager, - num_contexts=0, - batch_size=1, - num_accepted_tokens=torch.tensor([1], dtype=torch.int32), - original_all_rank_num_tokens=None, - ) - - assert events == ["switch", "refresh", "forward"] - # The draft forward raised, so only the ExitStack callback can have cleared - # this; a leaked flag would let the next target forward reuse the selections. - assert state == { - "in_mtp_draft_loop": False, - "mtp_num_accepted": None, - "skip_topk": False, - } - - -def test_dynamic_tree_metadata_forces_target_mask_prepare_each_step() -> None: +@pytest.mark.parametrize("dynamic", [True, False]) +def test_dynamic_tree_metadata_forces_target_mask_prepare_each_step( + dynamic: bool) -> None: metadata = TrtllmAttentionMetadata( seq_lens=None, seq_lens_kv=None, @@ -134,22 +63,15 @@ def test_dynamic_tree_metadata_forces_target_mask_prepare_each_step() -> None: max_num_requests=1, max_num_tokens=1, max_seq_len=1, + spec_decoding=SpecDecodingConfig( + is_spec_dec_tree=True, + is_spec_dec_dynamic_tree=dynamic, + max_draft_len=1, + max_total_draft_tokens=1, + ), ) - common_kwargs = dict( - batch_size=0, - is_spec_decoding_enabled=False, - is_spec_dec_tree=True, - max_draft_len=1, - max_total_draft_tokens=1, - ) - - metadata.update_spec_dec_param(is_spec_dec_dynamic_tree=True, - **common_kwargs) - assert metadata.force_prepare_spec_dec_tree_mask - - metadata.update_spec_dec_param(is_spec_dec_dynamic_tree=False, - **common_kwargs) - assert not metadata.force_prepare_spec_dec_tree_mask + metadata.update_spec_dec_param(batch_size=0) + assert metadata.force_prepare_spec_dec_tree_mask is dynamic def test_mtp_dynamic_tree_relocation_uses_full_attention_window( @@ -524,7 +446,9 @@ def test_external_shared_target_kv_step_prepares_next_new_tokens() -> None: spec_resource_manager=None, batch_indices_cuda=torch.tensor([0]), sharded_target_logits=False) - attn_metadata = SimpleNamespace(num_seqs=1, num_contexts=0) + attn_metadata = SimpleNamespace(num_seqs=1, + num_contexts=0, + find_ext=lambda extension_type: None) out = MTPEagleWorker._forward_impl(worker, torch.tensor([1, 2, 3]), @@ -649,8 +573,11 @@ def test_block_offsets_staging_width_spec_gate(spec_signal): ) mock_draft_manager = None if spec_signal == "draft_kv_cache_manager": + # A one-model draft over its own KV cache: the draft metadata stages + # the whole table of the draft manager; the target is not affected. mock_draft_manager = _make_mock_kv_cache_manager(num_seqs) metadata_kwargs["draft_kv_cache_manager"] = mock_draft_manager + metadata_kwargs["one_model_draft"] = True attn_metadata = TrtllmAttentionMetadata(**metadata_kwargs) if spec_signal == "is_spec_decoding_enabled": @@ -669,7 +596,7 @@ def test_block_offsets_staging_width_spec_gate(spec_signal): attn_metadata.prepare() - if spec_signal is None: + if spec_signal in (None, "draft_kv_cache_manager"): # Non-speculative: capped at ceil(max kv len / tokens_per_block). expected_max_blocks = -(-max(prompt_lens) // mock_kv_cache_manager.tokens_per_block) diff --git a/tests/unittest/_torch/speculative/test_force_accepted_tokens.py b/tests/unittest/_torch/speculative/test_force_accepted_tokens.py index f69a9788dc48..230e7773635f 100644 --- a/tests/unittest/_torch/speculative/test_force_accepted_tokens.py +++ b/tests/unittest/_torch/speculative/test_force_accepted_tokens.py @@ -35,7 +35,7 @@ import pytest import torch -from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata +from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata, BatchInputs from tensorrt_llm._torch.speculative.interface import ( FORCE_NUM_ACCEPTED_TOKENS_ENV_VAR, SpecWorkerBase, @@ -370,25 +370,33 @@ def test_cuda_graph_replay_advances_rng_state(): class _FailingSpecWorker(_StubSpecWorker): - """Stub whose forward saves spec-dec metadata state and then fails.""" + """Stub whose forward changes the draft's lengths in a draft loop and then fails.""" def _forward_impl(self, *args: object, **kwargs: object) -> None: attn_metadata = kwargs["attn_metadata"] - attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") - raise RuntimeError("simulated draft failure") + with self.draft_loop(attn_metadata) as draft: + draft.set_seq_lens(slice(0, 2), host=3, device=3, update_counts=True) + raise RuntimeError("simulated draft failure") -def test_forward_restores_spec_dec_state_on_failure() -> None: - """A failure between prepare_for_spec_dec and restore must not leak saved - attn-metadata state; SpecWorkerBase.forward restores it in its cleanup - (https://nvbugs/6442074).""" +def test_a_failed_draft_forward_leaves_the_target_as_prepared() -> None: + """A failure inside a draft loop must not leak into later forwards: the + loop closes, and the target's lengths, which the loop never changes, stay + as prepared (https://nvbugs/6442074).""" _require_cuda() - attn_metadata = AttentionMetadata(max_num_requests=2, max_num_tokens=16) - attn_metadata.seq_lens = torch.ones(2, dtype=torch.int32) + attn_metadata = AttentionMetadata(max_num_requests=2, max_num_tokens=16, one_model_draft=True) + attn_metadata.prepare( + BatchInputs( + seq_lens=torch.ones(2, dtype=torch.int32), + num_contexts=0, + request_ids=[0, 1], + prompt_lens=[1, 1], + ) + ) worker = _FailingSpecWorker() for _ in range(2): - # The second iteration would trip the pairing assert inside - # prepare_for_spec_dec if the first failure had leaked saved state. with pytest.raises(RuntimeError, match="simulated draft failure"): worker(attn_metadata=attn_metadata, spec_metadata=None) - assert not attn_metadata.has_spec_dec_saved_state + assert not attn_metadata.in_draft_loop + assert attn_metadata.seq_lens.tolist() == [1, 1] + assert not attn_metadata.lengths_changed diff --git a/tests/unittest/_torch/speculative/test_spec_tail_kernels.py b/tests/unittest/_torch/speculative/test_spec_tail_kernels.py index 2d216d2c88e2..ff41bee5f82e 100644 --- a/tests/unittest/_torch/speculative/test_spec_tail_kernels.py +++ b/tests/unittest/_torch/speculative/test_spec_tail_kernels.py @@ -15,7 +15,7 @@ """The spec-decode step kernels reproduce the eager chains they replace.""" import copy -from types import SimpleNamespace +from types import MethodType, SimpleNamespace import pytest import torch @@ -23,6 +23,7 @@ from _torch.helpers import triton_num_compiled from torch import nn +from tensorrt_llm._torch.attention.backends.core import AttentionCoreMetadata from tensorrt_llm._torch.models.modeling_dspark import _build_mla_block_fixup from tensorrt_llm._torch.speculative.dflash import DFlashWorker from tensorrt_llm._torch.speculative.spec_tail_kernels import ( @@ -264,13 +265,15 @@ def test_paged_latent_prepare_matches_eager(draft_len: int, based: bool) -> None ) def attn_metadata(): - return SimpleNamespace( + md = SimpleNamespace( num_contexts=num_ctx_reqs, num_seqs=batch, num_ctx_tokens=num_ctx_tokens, _seq_lens_cuda=torch.full((batch,), 99, dtype=torch.int32, device="cuda"), _seq_lens=torch.full((batch,), 99, dtype=torch.int32), ) + md.set_seq_lens = MethodType(AttentionCoreMetadata.set_seq_lens, md) + return md def prepare(fused): w = copy.copy(worker) diff --git a/tests/unittest/_torch/visual_gen/test_trtllm_attention_metadata.py b/tests/unittest/_torch/visual_gen/test_trtllm_attention_metadata.py index 56bd423cf611..47a67f279b7a 100644 --- a/tests/unittest/_torch/visual_gen/test_trtllm_attention_metadata.py +++ b/tests/unittest/_torch/visual_gen/test_trtllm_attention_metadata.py @@ -6,7 +6,9 @@ import pytest import torch +from _torch.helpers import bare_extension +from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata from tensorrt_llm._torch.attention.backends.interface import PredefinedAttentionMask from tensorrt_llm._torch.attention.backends.sparse.params import ( BlockSparseForwardInputs, @@ -61,13 +63,11 @@ def _make_core_forward_metadata(): metadata._seq_lens_kv = seq_lens metadata._seq_lens_cuda = None metadata.kv_cache_manager = None - metadata._max_seq_len_storage = 4 + metadata.max_seq_len = 4 metadata.use_paged_context_fmha = False metadata.cu_q_seqlens = None metadata.cu_kv_seqlens = None - metadata.enable_flash_mla = False - metadata.spec_bl_tree_first_sparse_mask_offset_kv = None - metadata.spec_decoding_bl_tree_mask = None + metadata.replace_ext(bare_extension(SpecDecodingMetadata, metadata)) metadata.kv_lens_cuda_runtime = torch.tensor([4], dtype=torch.int32) metadata.kv_lens_runtime = torch.tensor([4], dtype=torch.int32) metadata.prompt_lens_cuda_runtime = torch.tensor([4], dtype=torch.int32) diff --git a/tests/unittest/utils/metadata_contract.py b/tests/unittest/utils/metadata_contract.py new file mode 100644 index 000000000000..a0e743fc7700 --- /dev/null +++ b/tests/unittest/utils/metadata_contract.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared checks of the attention metadata extension contracts.""" + +from typing import Any, Dict, Optional + +import torch + +# Every production metadata extension and the test module that checks its +# creation configuration with ``check_copy_reproduces_creation_config``; None +# for an extension that takes no creation arguments. +CREATION_CONFIG_COVERAGE: Dict[str, Optional[str]] = { + "AttentionCoreMetadata": "attention/test_metadata_creation_config.py", + "VanillaCoreMetadata": "attention/test_metadata_creation_config.py", + "TrtllmCoreMetadata": "attention/test_metadata_creation_config.py", + "DraftMetadata": "attention/test_metadata_creation_config.py", + "SpecDecodingMetadata": "attention/test_metadata_creation_config.py", + "HelixMetadata": "attention/test_metadata_creation_config.py", + "FlashMlaMetadata": "attention/test_metadata_creation_config.py", + "MTPIndexShareExtension": "attention/test_metadata_creation_config.py", + "Qwen4ExpPleState": "attention/test_metadata_creation_config.py", + "CrossAttentionMetadata": "attention/test_metadata_creation_config.py", + "RecurrentMetadata": "attention/test_metadata_creation_config.py", + "FlashInferCoreMetadata": "attention/test_metadata_creation_config.py", + "Fp4MlaState": None, + "DSAMetadata": "attention/test_draft_metadata_reuse.py", + "GlmTp8DSAMetadata": "attention/test_draft_metadata_reuse.py", + "DeepseekV4Metadata": "attention/test_draft_metadata_reuse.py", + "CSA2Metadata": "attention/test_draft_metadata_reuse.py", + "QSAMetadata": "attention/sparse/qsa/test_qsa_cache_manager_layout.py", + "RocketMetadata": "attention/sparse/rocketkv/test_rocketkv.py", + "MiniMaxM3MsaMetadata": "attention/test_metadata_creation_config.py", + "MiniMaxM3TritonMetadata": "attention/test_metadata_creation_config.py", +} + + +def _same(expected: Any, actual: Any) -> bool: + if isinstance(expected, torch.Tensor) or isinstance(actual, torch.Tensor): + if expected is actual: + return True + return ( + isinstance(expected, torch.Tensor) + and isinstance(actual, torch.Tensor) + and expected.shape == actual.shape + and expected.dtype == actual.dtype + and torch.equal(expected.cpu(), actual.cpu()) + ) + if expected is actual: + return True + try: + return bool(expected == actual) + except (RuntimeError, TypeError, ValueError): + return False + + +def check_copy_reproduces_creation_config(metadata) -> None: + """A copy built from ``metadata``'s configuration reports that configuration. + + For every attached extension, the copy's ``creation_config()`` equals the + source's: each creation argument survives ``constructed_copy`` as given. + """ + copy = metadata.constructed_copy() + differences = {} + for extension in metadata._attached_extensions(): + expected = extension.creation_config() + actual = copy.ext(type(extension)).creation_config() + names = sorted( + name + for name in expected.keys() | actual.keys() + if not _same(expected.get(name), actual.get(name)) + ) + if names: + differences[type(extension).__name__] = names + assert differences == {}, f"creation arguments a constructed copy changed: {differences}" From 80322c191d89f563e91da72c578d7c0c82ce7a79 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 00:44:20 -0800 Subject: [PATCH 2/7] [None][feat] Keep main's metadata spelling where the extensions allow it Read extension state through the metadata, as main does, instead of through metadata.ext(T) / find_ext(T): - Declare the private names other code reads in their extension's fields or methods, so metadata._name forwards to the extension. - The core serves enable_flash_mla, enable_helix and fp4_mla_state from the attached extensions; the FP4 MLA helpers read metadata.fp4_mla_state again. - Drop the _dsa_state() accessors; MSA declares on_update_kv_lens and DSA bind_gen_indexer_kv_lens. Keep main's method order in flashinfer.py, trtllm.py, interface.py and the DSA and RocketKV metadata, so their diff shows only real changes. Signed-off-by: Yihan Wang --- .../modeling.py | 17 +- .../glm_5_2_nvfp4__sm_103__tp8/attention.py | 25 +- .../cross_rank_topk_global_emit/bridge.py | 16 +- .../kernels/rope_kv_append/ops.py | 16 +- .../_dsa_backend.py | 18 +- .../_indexer.py | 45 +- .../_torch/attention/backends/core.py | 24 + .../_torch/attention/backends/flashinfer.py | 2634 ++++++++--------- .../_torch/attention/backends/fmha/fp4_mla.py | 7 +- .../backends/fp4_mla/cache_gather.py | 8 +- .../backends/fp4_mla/cache_update.py | 68 +- .../attention/backends/fp4_mla/decode.py | 39 +- .../backends/fp4_mla/fp4_mla_context.py | 9 +- .../attention/backends/fp4_mla/layout.py | 9 +- .../attention/backends/fp4_mla/metadata.py | 117 +- .../attention/backends/fp4_mla/packed_plan.py | 10 +- .../attention/backends/fp4_mla/v_cache.py | 7 +- .../_torch/attention/backends/interface.py | 268 +- .../attention/backends/sparse/dsa/indexer.py | 50 +- .../attention/backends/sparse/dsa/metadata.py | 240 +- .../backends/sparse/minimax_m3/msa_backend.py | 1 + .../backends/sparse/rocket/metadata.py | 12 +- .../_torch/attention/backends/trtllm.py | 729 +++-- .../engine/runners/decoder/inputs.py | 3 +- tensorrt_llm/_torch/speculative/eagle3.py | 4 +- .../attention/sparse/msa/test_msa_backend.py | 16 +- 26 files changed, 2165 insertions(+), 2227 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py index c0afeb2d3eaa..15f81a293931 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py @@ -39,7 +39,6 @@ from torch import nn from transformers import PretrainedConfig -from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata from tensorrt_llm._torch.attention.backends.features.spec_decoding import set_use_spec_decoding from tensorrt_llm._torch.attention.backends.interface import ( AttentionForwardArgs, @@ -2383,17 +2382,19 @@ def _advance_lengths( attn_metadata.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) has_kv_cache = attn_metadata.kv_cache_manager is not None if has_kv_cache: - # The context requests continue as one-token generation rows. attn_metadata.as_generation_rows() - if num_contexts > 0 and attn_metadata.find_ext(FlashMlaMetadata) is not None: - attn_metadata.block_ids_per_seq[:batch_size].copy_( - attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True - ) attn_metadata.adjust_kv_lens( - slice(num_contexts, batch_size), - num_accepted_tokens[num_contexts:batch_size] - draft_len, + slice(num_contexts, batch_size), num_accepted_tokens[num_contexts:] - draft_len ) attn_metadata.adjust_kv_lens(slice(0, num_contexts), 1) + if ( + has_kv_cache + and num_contexts > 0 + and getattr(attn_metadata, "enable_flash_mla", False) + ): + attn_metadata.block_ids_per_seq[:batch_size].copy_( + attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True + ) target_metadata.refresh_draft() set_use_spec_decoding(attn_metadata, False) else: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py index f07e1292f35b..a01b42b429e6 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/attention.py @@ -30,10 +30,7 @@ ) from tensorrt_llm._torch.attention.backends.sparse.dsa.backend import DSATrtllmAttention from tensorrt_llm._torch.attention.backends.sparse.dsa.indexer import Indexer -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAMetadata, - DSAtrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata from tensorrt_llm._torch.attention.backends.sparse.dsa.module import DSAHooks from tensorrt_llm._torch.distributed import AllReduceParams from tensorrt_llm._torch.models.modeling_deepseekv3 import DeepseekV32Attention @@ -43,11 +40,6 @@ from .kernels.indexer_topk import region -def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: - """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" - return metadata.ext(DSAMetadata) - - def reclass(module, builtin: type, cls: type) -> None: """Re-class ``module``, which must be exactly the built-in class ``builtin``, as ``cls``. @@ -317,7 +309,6 @@ def sparse_attn_predict( forward_args: AttentionForwardArgs, ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: """Run the DSA indexer and convert its local TopK to paged indices.""" - dsa_state = _dsa_state(metadata) if dispatch.phase() is not dispatch.Phase.DECODE: return super().sparse_attn_predict( q=q, k=k, metadata=metadata, forward_args=forward_args @@ -369,7 +360,7 @@ def sparse_attn_predict( local_layer_idx = self.get_local_layer_idx(metadata) kv_cache_dtype = getattr(metadata.kv_cache_manager, "dtype", None) if kv_cache_dtype == tensorrt_llm.bindings.DataType.NVFP4 and not is_generation: - dsa_state._ensure_pool_view_cached() + metadata._ensure_pool_view_cached() page_index_scale, layer_offset = ( metadata.kv_cache_manager.get_primary_pool_page_index_params(local_layer_idx) ) @@ -399,19 +390,19 @@ def sparse_attn_predict( metadata.host_kv_cache_pool_pointers, metadata.host_kv_cache_pool_mapping, topk_indices, - dsa_state._cached_req_idx_ctx, - dsa_state._cached_block_table_ctx, + metadata._cached_req_idx_ctx, + metadata._cached_block_table_ctx, metadata.ctx_kv_indptr[: metadata.num_contexts + 1], scratch, compact_indices, self._nvfp4_mla_kv_scale_quant_orig, local_layer_idx, total_kv_tokens, - dsa_state._cached_tokens_per_block, - page_index_scale * dsa_state._cached_tokens_per_block, + metadata._cached_tokens_per_block, + page_index_scale * metadata._cached_tokens_per_block, layer_offset, metadata.kv_cache_manager.mla_kv_cache_residual_dim, - dsa_state._cached_num_pool_tokens, + metadata._cached_num_pool_tokens, ) metadata.nvfp4_mla_context_fp8_scratch = scratch forward_args.sparse_runtime_params.aux_kv_cache_pool_ptr = scratch.data_ptr() @@ -446,7 +437,7 @@ def sparse_attn_predict( self._nvfp4_mla_kv_scale_quant_orig, local_layer_idx, metadata.kv_cache_manager.mla_kv_cache_residual_dim, - dsa_state._cached_num_pool_tokens, + metadata._cached_num_pool_tokens, ) # Static sparse MLA treats indices as offsets from kvPtr. Feed it # compact offsets and the FP8 scratch base; TRTLLMGen is unchanged. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py index 45f36fe628c2..88afc55c7912 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/cross_rank_topk_global_emit/bridge.py @@ -55,16 +55,6 @@ ] -def _dsa_state(metadata): - """The DSA extension of ``metadata``, which holds the grouped-remap state. - - Imported on use, as the rest of this module's attention imports. - """ - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata - - return metadata.ext(DSAMetadata) - - class GlobalEmitContractError(RuntimeError): """The bridge work cross_rank_topk_global_emit performed at the top-k differs from what the call sites requested.""" @@ -75,7 +65,7 @@ class RemapPlan: ``grouped``: the leader of an active full + shared group remaps the whole group (``layer_ids``, ``page_index_scale``) and publishes it in - ``_dsa_state(metadata)._group_remap_batched[leader]``; the layer's own rows are + ``metadata._group_remap_batched[leader]``; the layer's own rows are ``out[slot]``. Otherwise the layer remaps alone (``page_index_scale``, ``layer_offset``). """ @@ -123,7 +113,7 @@ def predict_remap(metadata, local_layer: int) -> Optional[RemapPlan]: from tensorrt_llm._torch.attention.backends.sparse.dsa import backend as dsa_backend if dsa_backend._GROUP_REMAP and not metadata.in_mtp_draft_loop: - struct = _dsa_state(metadata)._ensure_group_remap_struct() + struct = metadata._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is not None and local_layer < len(leader_of): leader = leader_of[local_layer] @@ -291,6 +281,6 @@ def remap_topk_to_global(backend, topk: torch.Tensor, metadata, packet) -> torch packet.plan = None _check_plan(plan, backend, topk, metadata, packet) if plan.remap.grouped: - _dsa_state(metadata)._group_remap_batched[plan.remap.leader] = plan.pool + metadata._group_remap_batched[plan.remap.leader] = plan.pool return plan.pool[plan.remap.slot] return plan.pool diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py index ed0874744c8d..cd01fb563af6 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py @@ -49,16 +49,6 @@ _ROPE_DIM = 64 -def _dsa_state(metadata): - """The DSA extension of ``metadata``, which holds the grouped-remap state. - - Imported on use, as the rest of this module's attention imports. - """ - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSAMetadata - - return metadata.ext(DSAMetadata) - - @dataclass class IndexerKScatter: """Deferred native indexer K-cache scatter of one layer and one forward. @@ -483,7 +473,7 @@ def remap_topk_to_global( copy = packet.copy local_layer_idx = backend.get_local_layer_idx(metadata) if dsa_backend._GROUP_REMAP and not metadata.in_mtp_draft_loop: - struct = _dsa_state(metadata)._ensure_group_remap_struct() + struct = metadata._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is not None and local_layer_idx < len(leader_of): leader = leader_of[local_layer_idx] @@ -504,9 +494,9 @@ def remap_topk_to_global( page_index_scale=struct["group_scale"][leader], copy=copy, ) - _dsa_state(metadata)._group_remap_batched[leader] = batched + metadata._group_remap_batched[leader] = batched return batched[slot] - batched = _dsa_state(metadata)._group_remap_batched.get(leader, None) + batched = metadata._group_remap_batched.get(leader, None) if ( copy is None and batched is not None diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py index 2d61536381c8..61f1bc9126f6 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_dsa_backend.py @@ -15,10 +15,7 @@ MLAParams, PositionalEmbeddingParams, ) -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAMetadata, - DSAtrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata from tensorrt_llm._torch.attention.backends.sparse.dsa.params import DSAParams from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm.models.modeling_utils import QuantConfig @@ -29,12 +26,6 @@ transform_local_topk_and_prepare_pool_view_grouped, ) - -def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: - """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" - return metadata.ext(DSAMetadata) - - _GROUP_REMAP = os.environ.get("TRTLLM_DISABLE_DSA_GROUP_REMAP", "0") != "1" @@ -191,8 +182,7 @@ def _grouped_remap_topk_to_global( that equivalence is covered by unit tests (see ``tests/unittest/_torch/attention/sparse/dsa/test_cpp_custom_ops.py``). """ - dsa_state = _dsa_state(metadata) - struct = dsa_state._ensure_group_remap_struct() + struct = metadata._ensure_group_remap_struct() leader_of = struct.get("leader_of") if leader_of is None or local_layer_idx >= len(leader_of): return None @@ -210,7 +200,7 @@ def _grouped_remap_topk_to_global( struct["group_scale"][leader], is_generation, ) - dsa_state._group_remap_batched[leader] = batched + metadata._group_remap_batched[leader] = batched else: # Shared layer: read the leader's precomputed batch. The batch is # cleared at every step boundary, so a present entry is from this @@ -218,7 +208,7 @@ def _grouped_remap_topk_to_global( # top-k shape, and otherwise fall back to the per-layer path. This # makes any leader/follower ordering or shape mismatch a safe # per-layer fallback rather than a stale/out-of-bounds read. - batched = dsa_state._group_remap_batched.get(leader, None) + batched = metadata._group_remap_batched.get(leader, None) if ( batched is None or slot >= batched.shape[0] diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py index 2a28a41d94b2..90a2a92acad0 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_indexer.py @@ -13,10 +13,7 @@ import torch.nn.functional as F from tensorrt_llm._torch.attention.backends.interface import MLAParams, PositionalEmbeddingParams -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import ( - DSAMetadata, - DSAtrtllmAttentionMetadata, -) +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata from tensorrt_llm._torch.attention.backends.sparse.dsa.params import DSAParams from tensorrt_llm._torch.attention.backends.sparse.params import ( get_indexer_mqa_logits_elem_budget, @@ -39,12 +36,6 @@ from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig - -def _dsa_state(metadata: DSAtrtllmAttentionMetadata) -> DSAMetadata: - """The DSA extension of ``metadata``, which holds the indexer and pool-view state.""" - return metadata.ext(DSAMetadata) - - _INDEXER_MQA_LOGITS_ELEM_BUDGET = get_indexer_mqa_logits_elem_budget() @@ -70,32 +61,31 @@ def transform_local_topk_and_prepare_pool_view( Uses the pool-view values the DSA metadata caches (DSAMetadata._ensure_pool_view_cached) to avoid redundant Python/CUDA overhead across layers. """ - dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - dsa._ensure_pool_view_cached() + attn_metadata._ensure_pool_view_cached() page_index_scale, layer_offset = ( attn_metadata.kv_cache_manager.get_primary_pool_page_index_params(layer_idx) ) if is_generation: - block_table = dsa._cached_block_table_gen - req_idx = dsa._cached_req_idx_gen + block_table = attn_metadata._cached_block_table_gen + req_idx = attn_metadata._cached_req_idx_gen else: - block_table = dsa._cached_block_table_ctx - req_idx = dsa._cached_req_idx_ctx + block_table = attn_metadata._cached_block_table_ctx + req_idx = attn_metadata._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global( req_idx, block_table, topk_indices, - dsa._cached_tokens_per_block, + attn_metadata._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * dsa._cached_tokens_per_block, + page_index_scale * attn_metadata._cached_tokens_per_block, layer_offset, ) - return global_indices, dsa._cached_pool_view + return global_indices, attn_metadata._cached_pool_view def transform_local_topk_and_prepare_pool_view_grouped( @@ -120,25 +110,24 @@ def transform_local_topk_and_prepare_pool_view_grouped( is bit-identical to ``transform_local_topk_and_prepare_pool_view`` for the member with offset ``layer_ids[z]``. MLA-only path. """ - dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - dsa._ensure_pool_view_cached() + attn_metadata._ensure_pool_view_cached() if is_generation: - block_table = dsa._cached_block_table_gen - req_idx = dsa._cached_req_idx_gen + block_table = attn_metadata._cached_block_table_gen + req_idx = attn_metadata._cached_req_idx_gen else: - block_table = dsa._cached_block_table_ctx - req_idx = dsa._cached_req_idx_ctx + block_table = attn_metadata._cached_block_table_ctx + req_idx = attn_metadata._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global_grouped( req_idx, block_table, topk_indices, - dsa._cached_tokens_per_block, + attn_metadata._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * dsa._cached_tokens_per_block, + page_index_scale * attn_metadata._cached_tokens_per_block, layer_ids, ) @@ -998,7 +987,7 @@ def _store_mtp_shared_rows( ctx_rows = topk_indices_buffer[ctx_last] rows = ctx_rows if rows is None else torch.cat([ctx_rows, rows]) if rows is not None: - index_share = _dsa_state(metadata).index_share + index_share = metadata.index_share shared_topk_indices = index_share.shared_topk_indices if shared_topk_indices is None: index_share.bind_buffers(shared_topk_indices=rows.contiguous()) diff --git a/tensorrt_llm/_torch/attention/backends/core.py b/tensorrt_llm/_torch/attention/backends/core.py index 4f74e427b25b..f98f7da338d3 100644 --- a/tensorrt_llm/_torch/attention/backends/core.py +++ b/tensorrt_llm/_torch/attention/backends/core.py @@ -56,6 +56,9 @@ class AttentionCoreMetadata(MetadataExtension): "runtime_features", "is_cuda_graph", "enable_context_mla_with_cached_kv", + "enable_flash_mla", + "enable_helix", + "fp4_mla_state", "cache_indirection", "cuda_graph_buffers", "full_block_table_width", @@ -397,6 +400,27 @@ def num_seqs(self) -> int: """The number of sequences in the batch.""" return self.seq_lens.shape[0] + @property + def enable_flash_mla(self) -> bool: + """Whether FlashMLA serves the generation phase (its extension is attached).""" + from .features import FlashMlaMetadata + + return self.metadata.find_ext(FlashMlaMetadata) is not None + + @property + def enable_helix(self) -> bool: + """Whether Helix context parallelism is on (its extension is attached).""" + from .features import HelixMetadata + + return self.metadata.find_ext(HelixMetadata) is not None + + @property + def fp4_mla_state(self): + """The FP4 MLA extension when it is attached, else None.""" + from .fp4_mla.state import Fp4MlaState + + return self.metadata.find_ext(Fp4MlaState) + @property def is_cross(self) -> bool: """Whether this metadata is for cross attention.""" diff --git a/tensorrt_llm/_torch/attention/backends/flashinfer.py b/tensorrt_llm/_torch/attention/backends/flashinfer.py index 4c77a79e46cf..f2cb2c52914d 100644 --- a/tensorrt_llm/_torch/attention/backends/flashinfer.py +++ b/tensorrt_llm/_torch/attention/backends/flashinfer.py @@ -356,1428 +356,1034 @@ class FlashInferCoreMetadata(AttentionCoreMetadata): "workspace_buffer": None, } - def __init__(self, metadata, *, source=None, **config) -> None: - super().__init__(metadata, source=source, **config) - # MLA wrappers and stable buffers. - # Cached plan params + is-planned flag let prepare() refresh the plan - # outside stream capture (flashinfer plan() does device->host syncs). - self._ragged_prefill_wrapper = None - self._mla_decode_wrapper = None - self._mla_context_wrapper = None - self._mla_ragged_plan_params = None - self._mla_context_plan_params = None - self._mla_decode_plan_params = None - self.num_ctx_cached_tokens = 0 - self._mla_ragged_planned = False - self._mla_context_planned = False - self._mla_decode_planned = False - self._mla_qo_indptr_buf = None - # Graph-stable page-index buffer of each VSWA pool, keyed by pool ID. CUDA - # graph copies share their source's buffers; a separate-KV draft view - # allocates its own. - self._vswa_pool_bufs = self.shared_or("_vswa_pool_bufs", lambda: dict()) - self._mla_kv_len_arr_buf = None - self._multi_item_params = None - self._draft_metadata = None - self._is_shared_kv_draft_view = False - self._is_separate_kv_draft_view = False - self._uses_full_draft_page_table = False - # Whether target generation rows expose reserved speculative pages while keeping their - # attention-visible lengths in `_logical_kv_lens`. - self._uses_full_generation_page_table = False - self.workspace_buffer = self.shared_or("workspace_buffer", - lambda: self.workspace_buffer) - buffers = metadata.cuda_graph_buffers - md = self.metadata - capture_graph = self.is_cuda_graph + def needs_plan(self, plan_params: PlanParams) -> bool: + if plan_params not in self._plan_params_to_wrappers: + return True - if self.workspace_buffer is None: - # Note: even though flashinfer only recommends 128 MB, we have to push it - # a bit higher to cover all possible CUDA graph cases. If it's too small, - # warmup will crash. - # FlashInfer's split-KV decode plan sizes its temp buffers proportionally to - # SM count; 320 MB was tuned for a 148-SM baseline, so scale up (never down) - # on GPUs with more SMs. - baseline_sms = 148 - baseline_bytes = 320 * 1024 * 1024 - num_sms = torch.cuda.get_device_properties( - torch.cuda.current_device()).multi_processor_count - workspace_bytes = max(baseline_bytes, - baseline_bytes * num_sms // baseline_sms) - self.workspace_buffer = md.get_empty( - buffers, - (workspace_bytes, ), - dtype=torch.uint8, - cache_name="workspace_buffer", - capture_graph=capture_graph, + wrappers = self._plan_params_to_wrappers[plan_params] + return not wrappers.is_planned + + def get_prefill_wrapper( + self, plan_params: PlanParams + ) -> flashinfer.BatchPrefillWithPagedKVCacheWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[plan_params].prefill_wrapper + assert result is not None, "Prefill wrapper was not created in plan()" + return result + + def get_decode_wrapper( + self, plan_params: PlanParams + ) -> flashinfer.BatchDecodeWithPagedKVCacheWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[plan_params].decode_wrapper + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + if result._backend != "trtllm-gen": + raise ValueError( + "FlashInfer draft metadata views require the trtllm-gen " + "decode backend.") + self._publish_decode_wrapper_kv_lens(result) + return result + + def get_ragged_prefill_wrapper( + self, plan_params: PlanParams) -> _RaggedPrefillWrapper: + assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" + result = self._plan_params_to_wrappers[ + plan_params].ragged_prefill_wrapper + assert result is not None, "Ragged prefill wrapper was not created in plan()" + return result + + def plan_ragged( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + plan_params: RaggedPlanParams, + ) -> flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper: + """Plan MLA ragged prefill with expanded K, V (not paged).""" + if self._ragged_prefill_wrapper is None: + self._ragged_prefill_wrapper = flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper( + self.workspace_buffer, + "NHD", ) + if self._mla_ragged_plan_params != plan_params: + self._mla_ragged_planned = False + self._mla_ragged_plan_params = plan_params - self.paged_kv_indptr_decode = torch.empty((self.max_num_requests + 1, ), - device='cuda', - dtype=torch.int) - self.paged_kv_indptr_prefill = torch.empty( - (self.max_num_requests + 1, ), device='cuda', dtype=torch.int) - self._paged_kv_last_page_len = torch.empty((self.max_num_requests, ), - device='cuda', - dtype=torch.int) + if self._mla_ragged_planned: + return self._ragged_prefill_wrapper - self._qo_indptr = torch.zeros(self.max_num_requests + 1, - device='cuda', - dtype=torch.int) + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise ValueError( + "Cannot plan() flashinfer MLA ragged prefill while the stream " + "is capturing. Make sure prepare() has run at least one " + "warmup forward pass before capture.") - self._kv_indptr = torch.zeros( - self.max_num_requests + 1, device='cuda', - dtype=torch.int) if self.is_cross else self._qo_indptr + # Split append_paged_mla_kv_cache from plan() when this wrapper needs a + # new plan. Reusing a cached plan avoids this sync on later layers. + torch.cuda.current_stream().synchronize() - self._ragged_qo_indptr_buf = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_ragged_qo_indptr_buf", - capture_graph=capture_graph, - ) - self._ragged_kv_indptr_buf = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_ragged_kv_indptr_buf", - capture_graph=capture_graph, - ) + self._do_plan_ragged(qo_indptr, kv_indptr, plan_params) + self._mla_ragged_planned = True - self._cached_token_lens = torch.empty((self.max_num_requests, ), - dtype=torch.int, - device='cuda') - self._logical_kv_lens = md.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int, - cache_name="_logical_kv_lens", - capture_graph=capture_graph, - ) - self._draft_kv_runtime_lens = md.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int, - cache_name="_draft_kv_runtime_lens", - capture_graph=capture_graph, + return self._ragged_prefill_wrapper + + def _do_plan_ragged( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + plan_params: RaggedPlanParams, + ) -> None: + assert self._ragged_prefill_wrapper is not None + self._ragged_prefill_wrapper.plan( + qo_indptr, + kv_indptr, + plan_params.num_heads, + plan_params.num_kv_heads, + plan_params.head_dim, + head_dim_vo=plan_params.head_dim_vo, + use_fp16_qk_reduction=False, + causal=True, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + sm_scale=plan_params.sm_scale, ) - self._batch_indices = torch.empty((self.max_num_tokens, ), - dtype=torch.int, - device='cuda') - self._positions = torch.empty((self.max_num_tokens, ), - dtype=torch.int, - device='cuda') - self._plan_params_to_wrappers = {} - # Host-side state for sync-free trtllm-gen decode plans, retained by - # prepare(): a host mirror of _paged_kv_indices (kept in lockstep - # with the device buffer, with per-pool copies for VSWA swaps) and a - # host copy of the decode indptr. The per-plan_params persistent - # block-table buffers live on FlashInferWrappers. - self._host_pool_indices: Dict[int, torch.Tensor] = {} - self._host_paged_kv_indices: Optional[torch.Tensor] = None - self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None - self._uses_full_generation_page_table = False - self._max_num_blocks_per_seq = 0 + def plan_mla_decode( + self, + plan_params: MLAPlanParams, + ) -> object: + """Plan MLA decode using BatchMLAPagedAttentionWrapper. - # VSWA (Variable Sliding Window Attention): models with per-layer - # max_attention_window create separate V2 pool groups with independent - # page numbering. We need per-pool paged_kv_indices so each layer can - # use the indices that match its pool's buffer. - self._vswa_layer_to_pool: Optional[Dict[int, int]] = None - self._vswa_pool_indices_cache: Optional[Dict[int, torch.Tensor]] = None - self._primary_kv_layer_idx: Optional[int] = None - self._primary_kv_pool_id = 0 + Caches the wrapper and plan params; the actual plan() call is driven + by prepare() so it runs outside of CUDA graph capture. + """ + if self._mla_decode_wrapper is None: + self._mla_decode_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=self.is_cuda_graph, + qo_indptr=self._mla_qo_indptr_buf, + kv_indptr=self.paged_kv_indptr_decode, + kv_indices=self._paged_kv_indices, + kv_len_arr=self._mla_kv_len_arr_buf, + backend="auto", + ) - if self.kv_cache_manager is not None: - blocks_in_primary_pool = self.kv_cache_manager.blocks_in_primary_pool - attention_layer_indices = _get_attention_layer_indices( - self.kv_cache_manager) + if self._mla_decode_plan_params != plan_params: + self._mla_decode_planned = False - # Maximum block count across attention pools: sizes the shared - # page-index buffer and the VSWA pool buffers below. - max_num_blocks = blocks_in_primary_pool - for layer_idx in attention_layer_indices: - layer_buffer = self.kv_cache_manager.get_buffers(layer_idx) - if layer_buffer is not None: - max_num_blocks = max(max_num_blocks, layer_buffer.shape[0]) - if self._has_mixed_precision_staging(): - # The staging page space spans every sequence slot. - max_num_blocks = max( - max_num_blocks, - (self.max_num_sequences or self.max_num_requests) * - self.kv_cache_manager.max_blocks_per_seq) - self._paged_kv_indices = md.get_empty( - buffers, - (max_num_blocks, ), - dtype=torch.int, - cache_name="_paged_kv_indices", - capture_graph=capture_graph, - ) - self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq + # Cache params so prepare() can re-plan on subsequent forward passes. + self._mla_decode_plan_params = plan_params - # Layers may share one page-index list only when they are in the - # same pool, page-index scale, and KV factor: VSWA splits pools, - # and per-layer geometry (e.g. Gemma4 sliding/global head_dim) - # splits scales even when all windows collapse to max_seq_len and - # is_vswa is False. Mixed K-only/KV layers can use different view - # indices even with the same physical scale. Guarded on V2-specific - # attributes (V1 managers lack the per-pool infrastructure). - mgr = self.kv_cache_manager - get_scale = getattr(mgr, 'get_layer_page_index_scale', None) - get_kv_factor = getattr(mgr, 'get_layer_kv_factor', None) - layer_space: Dict[int, int] = {} - if hasattr(mgr, 'layer_to_pool_mapping_dict') and get_scale: - space_ids = {} - for layer_idx in attention_layer_indices: - layer_offset = mgr.layer_offsets[layer_idx] - key = (mgr.layer_to_pool_mapping_dict[layer_offset], - get_scale(layer_idx), - get_kv_factor(layer_idx) if get_kv_factor else None) - if (self._has_mixed_precision_staging() and layer_offset - in mgr.mixed_precision_high_layers): - key = ("mixed_precision_staging", ) - space_ids.setdefault(key, len(space_ids)) - layer_space[layer_idx] = space_ids[key] - if layer_space: - self._primary_kv_layer_idx = next(iter(layer_space)) - self._primary_kv_pool_id = layer_space[ - self._primary_kv_layer_idx] - if layer_space and (getattr(mgr, 'is_vswa', False) - or len(set(layer_space.values())) > 1): - self._vswa_layer_to_pool = {} - self._vswa_pool_to_rep_layer: Dict[int, int] = {} - for layer_idx, pool_id in layer_space.items(): - self._vswa_layer_to_pool[layer_idx] = pool_id - if pool_id not in self._vswa_pool_to_rep_layer: - self._vswa_pool_to_rep_layer[pool_id] = layer_idx - # Pre-allocate VSWA pool cache buffers. These must be - # stable (never reallocated) so that CUDA-graph-recorded - # copies reference valid addresses across replays. - # max_num_blocks (computed above) covers ALL pools so that - # secondary pool buffers are large enough. - for pool_id in set(self._vswa_layer_to_pool.values()): - if self._vswa_pool_bufs.get(pool_id) is None: - self._vswa_pool_bufs[pool_id] = torch.empty( - max_num_blocks, dtype=torch.int, device='cuda') - # Stable buffers for FlashInfer MLA decode; required for CUDA graphs. - self._mla_qo_indptr_buf = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - dtype=torch.int32, - cache_name="_mla_qo_indptr_buf", - capture_graph=capture_graph, - ) - self._mla_kv_len_arr_buf = md.get_empty( - buffers, - (self.max_num_requests, ), - dtype=torch.int32, - cache_name="_mla_kv_len_arr_buf", - capture_graph=capture_graph, - ) - # Rebind the wrapper to the freshly allocated buffers. - self._ragged_prefill_wrapper = None - self._mla_decode_wrapper = None - self._mla_context_wrapper = None - self._mla_ragged_plan_params = None - self._mla_context_plan_params = None - self._mla_decode_plan_params = None - self._mla_ragged_planned = False - self._mla_context_planned = False - self._mla_decode_planned = False - self._multi_item_params = None - - def derive(self) -> None: - md = self.metadata + if self._mla_decode_planned: + return self._mla_decode_wrapper - def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: - """Host int32 staging tensor for async H2D copies; pinned when - beneficial. + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise ValueError( + "Cannot plan() flashinfer MLA decode while the stream is " + "capturing. Make sure prepare() has run at least one warmup " + "forward pass before capture.") - torch.tensor(..., pin_memory=True) rejects numpy sources, hence - from_numpy + pin. - """ - return maybe_pin_memory( - torch.from_numpy(arr.astype(np.int32, copy=False))) + # Split append_paged_mla_kv_cache from plan() on a cache miss. prepare() + # calls _do_plan_mla_decode() directly and does not need this sync. + torch.cuda.current_stream().synchronize() + self._do_plan_mla_decode(plan_params) + self._mla_decode_planned = True + return self._mla_decode_wrapper - extra_attrs = get_model_extra_attrs() - if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(md) - # start and end indices of each sequence in the ragged query - assert self.seq_lens_cuda is not None - torch.cumsum(self.seq_lens_cuda, - dim=0, - dtype=torch.int32, - out=self._qo_indptr[1:self.seq_lens_cuda.size(0) + 1]) + def plan_mla_context( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_last_page_len: torch.Tensor, + plan_params: MLAPlanParams, + ) -> object: + """Plan MLA context with cached KV using BatchMLAPagedAttentionWrapper.""" + if self._mla_context_wrapper is None: + self._mla_context_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=False, + backend="auto", + ) - if self.multi_item_part_lens is not None: - self._multi_item_params = self._process_multi_item_part_lens( - self.multi_item_part_lens, device=self.seq_lens_cuda.device) - else: - self._multi_item_params = None + if self._mla_context_plan_params != plan_params: + self._mla_context_planned = False + self._mla_context_plan_params = plan_params - if self.kv_cache_manager is None: - assert self.request_ids is not None - assert self.num_generations == 0, ( - "FlashInfer without a KV cache manager only supports context-only " - "batches (num_generations == 0) in TRT-LLM.") - if self.is_cross: - raise NotImplementedError( - "FlashInfer without a KV cache manager is not tested for cross attention." - ) - self.kv_cache_params = KVCacheParams(use_cache=False) - n = self.num_seqs - self._cached_token_lens[:n].zero_() - self.num_ctx_cached_tokens = 0 - self._clean_cached_plans(defer_plan=False) - return + if self._mla_context_planned: + return self._mla_context_wrapper - if self._multi_item_params is not None: + if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): raise ValueError( - "multi_item_part_lens with KV cache is not supported") - - # number of tokens in the kv cache for each sequence in the batch - num_cached_tokens_per_seq = self.kv_cache_params.num_cached_tokens_per_seq - cached_token_lens = torch.tensor(num_cached_tokens_per_seq, - dtype=torch.int, - pin_memory=prefer_pinned()) - self._cached_token_lens[:cached_token_lens.size(0)].copy_( - cached_token_lens, non_blocking=True) - if self.num_contexts > 0: - self.num_ctx_cached_tokens = sum( - num_cached_tokens_per_seq[:self.num_contexts]) - else: - self.num_ctx_cached_tokens = 0 + "Cannot plan() flashinfer MLA context while the stream is " + "capturing. Make sure prepare() has run at least one warmup " + "forward pass before capture.") - # Number of tokens needed in the KV cache for each sequence after the next pass. - # Kept on the host: every consumer below needs host values, so a device-side computation - # would force a sync per step. These logical block counts describe committed history, not - # all pages that the KV manager has reserved for fixed-width speculative appends. - kv_lens_host = np.asarray(num_cached_tokens_per_seq, - dtype=np.int64) + self.seq_lens_kv.numpy() - logical_num_blocks = ((kv_lens_host + self.page_size - 1) // - self.page_size) - num_blocks = logical_num_blocks - use_full_generation_page_table = ( - self.kv_cache_params.use_full_generation_page_table) - self._uses_full_generation_page_table = use_full_generation_page_table - if use_full_generation_page_table: - # Generation rows need every reserved page to keep a speculative append addressable - # across a page boundary. Treating that wider table as the logical sequence would make - # FlashInfer attend to uncommitted draft slots, so plans and positions use the separate - # device logical lengths populated below. - assert self.request_ids is not None - num_blocks = np.asarray( - _get_page_table_num_blocks( - self.kv_cache_manager, - self.request_ids, - logical_num_blocks.tolist(), - self.num_contexts, - ), - dtype=np.int64, - ) - self.num_blocks = num_blocks.tolist() + # Split append_paged_mla_kv_cache from plan() when this wrapper needs a + # new plan. Reusing a cached plan avoids this sync on later layers. + torch.cuda.current_stream().synchronize() - assert self.request_ids is not None + self._do_plan_mla_context( + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + plan_params, + ) + self._mla_context_planned = True - # start and end indices of each sequence in the ragged key and value - # for md attention it's the same as qo_indptr so avoid computing twice. - if self.is_cross: - # start and end indices of each sequence in the ragged key and value - torch.cumsum(self.seq_lens_kv_cuda, - dim=0, - dtype=torch.int32, - out=self._kv_indptr[1:self.seq_lens_kv_cuda.size(0) + - 1]) + return self._mla_context_wrapper - # number of cache blocks used by each sequence in the cache - # NOTE: do not use len(block_ids) - that will give you a number - # that can be too big if using chunked prefill/kv cache reuse - # since we allocate all blocks ahead of time. - self.num_context_blocks = sum(self.num_blocks[:self.num_contexts]) - self.num_generation_blocks = sum(self.num_blocks[self.num_contexts:]) + def _do_plan_mla_context( + self, + qo_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_last_page_len: torch.Tensor, + plan_params: MLAPlanParams, + ) -> None: + assert self._mla_context_wrapper is not None - if self._has_mixed_precision_staging(): - self.prepare_mixed_precision_kv( - self.kv_layout, self.kv_cache_manager.tokens_per_block) + num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] + kv_len_arr = (num_pages_per_seq - + 1) * plan_params.page_size + kv_last_page_len - # indices of used cache blocks for each sequence - primary_layer_idx = self._primary_kv_layer_idx - kv_cache_manager = (self.get_mixed_precision_kv_view(primary_layer_idx) - or self.kv_cache_manager) - paged_kv_indices = kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, self.num_blocks, layer_idx=primary_layer_idx) - if primary_layer_idx is not None: - self._sanitize_swa_page_indices(paged_kv_indices, primary_layer_idx) + self._mla_context_wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + plan_params.num_heads, + plan_params.kv_lora_rank, + plan_params.qk_rope_head_dim, + plan_params.page_size, + causal=True, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + sm_scale=plan_params.sm_scale, + ) - self._paged_kv_indices[:paged_kv_indices.size(0)].copy_( - paged_kv_indices, non_blocking=True) + def _do_plan_mla_decode(self, plan_params: MLAPlanParams) -> None: + """Compute MLA decode plan inputs and call wrapper.plan(). - # Retain a host mirror of _paged_kv_indices: decode plans build the - # trtllm-gen block tables from host data, with no GPU round trips. - # The VSWA block below re-points the mirror at the active pool's - # copy whenever the device buffer is swapped. - self._host_paged_kv_indices = paged_kv_indices + Must run outside of CUDA graph capture. + """ + num_gen = self.num_generations + kv_indptr = self.paged_kv_indptr_decode[:num_gen + 1] + kv_indices = self._paged_kv_indices[self.num_context_blocks:self. + num_context_blocks + + self.num_generation_blocks] + kv_last_page = self._paged_kv_last_page_len[self.num_contexts:self. + num_contexts + num_gen] - # VSWA: build per-pool page index CUDA tensors so each layer can use - # the indices that match its own pool's buffer. Tensors live on CUDA - # so that forward_impl swap via copy_() is device-to-device (CUDA-graph - # capturable). - if self._vswa_layer_to_pool is not None: - unique_pools = set(self._vswa_layer_to_pool.values()) - primary_pool_id = self._primary_kv_pool_id - # Use dedicated pre-allocated buffers for each pool's indices. - # These buffers are created in __post_init__ so their addresses - # stay stable across CUDA-graph replays. - total_idx = paged_kv_indices.size(0) - primary_buf = self._vswa_pool_bufs[primary_pool_id] - primary_buf[:total_idx].copy_(self._paged_kv_indices[:total_idx], - non_blocking=True) - self._vswa_pool_indices_cache = { - primary_pool_id: primary_buf, - } - self._host_pool_indices = {primary_pool_id: paged_kv_indices} - for pool_id in unique_pools: - if pool_id == primary_pool_id: - continue - rep_layer = self._vswa_pool_to_rep_layer[pool_id] - kv_cache_manager = (self.get_mixed_precision_kv_view(rep_layer) - or self.kv_cache_manager) - pool_indices = \ - kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, self.num_blocks, layer_idx=rep_layer) - self._sanitize_swa_page_indices(pool_indices, rep_layer) - buf = self._vswa_pool_bufs[pool_id] - buf[:pool_indices.size(0)].copy_(pool_indices, - non_blocking=True) - self._vswa_pool_indices_cache[pool_id] = buf - self._host_pool_indices[pool_id] = pool_indices - self._vswa_active_pool_id = primary_pool_id + # _qo_indptr is ordered [context_seqs..., generation_seqs...]; rebase + # the generation slice to 0. + num_ctx = self.num_contexts + qo_indptr = self._qo_indptr[num_ctx:num_ctx + num_gen + + 1] - self._qo_indptr[num_ctx] - # number of tokens in the last cache block used by each sequence, - # derived from the logical rather than reservation-width page count. - paged_kv_last_page_len = _to_int32_tensor(kv_lens_host - - (logical_num_blocks - 1) * - self.page_size) - self._paged_kv_last_page_len[:paged_kv_last_page_len.size(0)].copy_( - paged_kv_last_page_len, non_blocking=True) + if self._uses_full_generation_page_table: + kv_len_arr = self._logical_kv_lens[num_ctx:num_ctx + num_gen] + else: + num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] + kv_len_arr = (num_pages_per_seq - + 1) * plan_params.page_size + kv_last_page - # Ragged page table, see https://docs.flashinfer.ai/tutorials/kv_layout.html#page-table-layout - # For decoding, this MUST be allocated ahead of time (for CUDA graphs). - # Prefill is prepared here as well just for the sake of consistency. - paged_kv_indptr_decode = _to_int32_tensor( - np.concatenate([[0], np.cumsum(num_blocks[self.num_contexts:])])) - self.paged_kv_indptr_decode[:paged_kv_indptr_decode.size(0)].copy_( - paged_kv_indptr_decode, non_blocking=True) - # Retain the host copy: decode plans hand it to flashinfer so that - # its indptr.cpu()/get_seq_lens calls do no D2H work. - self._host_paged_kv_indptr_decode = paged_kv_indptr_decode + self._mla_decode_wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + plan_params.num_heads, + plan_params.kv_lora_rank, + plan_params.qk_rope_head_dim, + plan_params.page_size, + causal=True, + q_data_type=plan_params.q_dtype, + kv_data_type=plan_params.kv_dtype, + sm_scale=plan_params.sm_scale, + ) - paged_kv_indptr_prefill = _to_int32_tensor( - np.concatenate([[0], np.cumsum(num_blocks[:self.num_contexts])])) - self.paged_kv_indptr_prefill[:paged_kv_indptr_prefill.size(0)].copy_( - paged_kv_indptr_prefill, non_blocking=True) + @property + def paged_kv_indices(self) -> torch.Tensor: + return self._paged_kv_indices[:self.num_generation_blocks + + self.num_context_blocks] - # This paged_kv_indptr attribute has both prefill and decode information in it. - # It's for the append_paged_kv_cache kernel. - if self.num_contexts == 0: - self.paged_kv_indptr = self.paged_kv_indptr_decode[: - paged_kv_indptr_decode - .size(0)] - elif self.num_generations == 0: - self.paged_kv_indptr = self.paged_kv_indptr_prefill[: - paged_kv_indptr_prefill - .size(0)] - else: - assert not self.is_cuda_graph, "Cannot mix decode/prefill with CUDA graphs" - # Accumulate on the host and stage through pinned memory: .cuda() - # on an unpinned tensor is a synchronous H2D that stalls the - # executor thread behind in-flight kernels on every mixed step. - self.paged_kv_indptr = _to_int32_tensor( - np.concatenate([[0], - np.cumsum(num_blocks)])).to(device='cuda', - non_blocking=True) + def get_paged_kv_indices_for_layer(self, layer_idx: int) -> torch.Tensor: + """Return page indices for the pool that *layer_idx* belongs to. - # For cross attention, num_tokens is 0 during decode, and we don't need to update kv cache. - if self.num_tokens > 0: - if use_full_generation_page_table: - # The page-table indptr now describes addressable capacity. Deriving positions from - # it would count reserved pages as committed KV and potentially shift appends beyond - # the true sequence end. - logical_kv_lens = _to_int32_tensor(kv_lens_host) - self._logical_kv_lens[:logical_kv_lens.numel()].copy_( - logical_kv_lens, non_blocking=True) - position_kv_lens = self._logical_kv_lens[:self.num_seqs] - else: - position_kv_lens = flashinfer.get_seq_lens( - self.paged_kv_indptr, - self.paged_kv_last_page_len, - self.page_size, - ) - batch_indices, positions = flashinfer.get_batch_indices_positions( - self.kv_indptr, - position_kv_lens, - self.num_tokens, - ) - self._batch_indices[:batch_indices.size(0)].copy_(batch_indices, - non_blocking=True) - self._positions[:positions.size(0)].copy_(positions, - non_blocking=True) + For non-VSWA models this returns the default shared indices. + For VSWA models it returns the pool-specific indices. + """ + if (self._vswa_layer_to_pool is None + or self._vswa_pool_indices_cache is None): + return self.paged_kv_indices + pool_id = self._vswa_layer_to_pool.get(layer_idx) + if pool_id is None: + return self.paged_kv_indices + total_blocks = self.num_generation_blocks + self.num_context_blocks + return self._vswa_pool_indices_cache[pool_id][:total_blocks] - # Re-plan MLA wrappers outside of forward/capture using the params - # cached by prior warmup forwards. Forward still handles first-use or - # dtype/shape changes by syncing only on a plan cache miss. - self._mla_ragged_planned = False - self._mla_context_planned = False - self._mla_decode_planned = False - if (self.num_contexts > 0 and self._mla_ragged_plan_params is not None - and self._ragged_prefill_wrapper is not None): - ragged_indptr = self.qo_indptr[:self.num_contexts + 1] - self._do_plan_ragged(ragged_indptr, ragged_indptr, - self._mla_ragged_plan_params) - self._mla_ragged_planned = True + def _sanitize_swa_page_indices(self, page_indices: torch.Tensor, + layer_idx: int) -> None: + """Replace evicted SWA pages with a safe in-range page index.""" + window_vec = getattr(self.kv_cache_manager, 'max_attention_window_vec', + None) + if not window_vec: + return + local_layer_idx = self.kv_cache_manager.layer_offsets[layer_idx] + if window_vec[local_layer_idx] is None: + return - if (self.num_contexts > 0 and self._mla_context_plan_params is not None - and self._mla_context_wrapper is not None): - num_contexts = self.num_contexts - num_context_blocks = self.num_context_blocks - context_qo_indptr = self.qo_indptr[:num_contexts + 1] - context_kv_indptr = self.paged_kv_indptr_prefill[:num_contexts + 1] - context_kv_indices = self._paged_kv_indices[:num_context_blocks] - context_last_page_len = self._paged_kv_last_page_len[:num_contexts] - self._do_plan_mla_context( - qo_indptr=context_qo_indptr, - kv_indptr=context_kv_indptr, - kv_indices=context_kv_indices, - kv_last_page_len=context_last_page_len, - plan_params=self._mla_context_plan_params, - ) - self._mla_context_planned = True + # KVCacheManagerV2 marks evicted out-of-window pages with -1. + # FlashInfer may dereference page IDs before applying window_left, so + # keep masked positions in range. The SWA mask excludes their values + # from the attention result. + # + # Page 0 is a live page owned by whichever request holds it, so parking + # there has the kernel dereference a stranger's keys and values, with + # the mask as the only thing keeping them out of the result. When the + # KV cache manager has reserved a guard page (TRTLLM_KV_GUARD_PAGE), + # park on that instead: a page no request can own, holding a known + # pattern. The default -- no guard page -- keeps page 0. + page_indices.masked_fill_(page_indices == BAD_PAGE_INDEX, + self._swa_park_page(layer_idx)) - # Re-plan the MLA decode wrapper outside of any stream capture. - if (self.num_generations > 0 - and self._mla_decode_plan_params is not None - and self._mla_decode_wrapper is not None): - self._do_plan_mla_decode(self._mla_decode_plan_params) - self._mla_decode_planned = True + def _swa_park_page(self, layer_idx: int) -> int: + """Page index that masked-out entries are pointed at for this layer.""" + guard = getattr(self.kv_cache_manager, 'guard_page_index', None) + if guard is None: + return 0 + page = guard(layer_idx) + return 0 if page is None else int(page) - # VSWA: restore primary pool indices as the default. - if (self._vswa_layer_to_pool is not None - and self._vswa_pool_indices_cache is not None): - primary_pool_id = self._primary_kv_pool_id - total_blocks = self.num_generation_blocks + self.num_context_blocks - src = self._vswa_pool_indices_cache[primary_pool_id][:total_blocks] - self._paged_kv_indices[:total_blocks].copy_(src, non_blocking=True) - self._vswa_active_pool_id = primary_pool_id - self._host_paged_kv_indices = \ - self._host_pool_indices[primary_pool_id] + def swap_paged_kv_indices_for_layer(self, layer_idx: int) -> None: + """Copy pool-specific page indices into the shared buffer. - self._debug_validate_page_table(kv_lens_host, logical_num_blocks, - num_blocks) + The FlashInfer wrappers reference ``_paged_kv_indices`` directly, + so we overwrite its contents with the correct pool's data before + each layer's plan/run cycle. We track the currently active pool + so we only copy when the pool actually changes. - # CUDA graph + trtllm-gen: update _block_tables and _kv_lens_buffer - # so the trtllm-gen decode kernel uses current page indices. - if (self.is_cuda_graph and self._vswa_layer_to_pool is not None - and self._vswa_pool_indices_cache is not None - and self.num_generations > 0): - decode_blocks = num_blocks[self.num_contexts:] - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - if plan_params.attention_mask_data is not None: - continue - decode_wrapper = wrappers.decode_wrapper - block_tables = getattr(decode_wrapper, '_block_tables', None) - if block_tables is None: - continue - pool_id = plan_params.kv_pool_id - if pool_id is None: - continue - batch_size, table_width = block_tables.shape - rows = min(batch_size, self.num_generations) - num_blocks_per_row = decode_blocks[:rows] - active_width = min(int(num_blocks_per_row.max()), table_width) - # Include the previous live extent once when the batch or its longest sequence - # shrinks. The zero-filled copy below clears entries that are no longer live; the - # next update can shrink. - update_rows = min( - batch_size, - max(rows, wrappers.decode_block_table_active_rows)) - update_width = min( - table_width, - max(active_width, wrappers.decode_block_table_active_width)) - host_block_tables = wrappers.host_decode_block_tables - if (host_block_tables is None - or host_block_tables.size(0) < update_rows - or host_block_tables.size(1) < update_width): - # Grow geometrically to avoid reallocating whenever the longest sequence - # acquires one more KV-cache block. - host_width = min( - max(64, 1 << (update_width - 1).bit_length()), - table_width) - host_block_tables = torch.zeros( - (self.max_num_requests, host_width), - dtype=torch.int32, - pin_memory=prefer_pinned()) - wrappers.host_decode_block_tables = host_block_tables + """ + if self._vswa_layer_to_pool is None: + return + pool_id = self._vswa_layer_to_pool.get(layer_idx) + if pool_id is None: + return # Layer not in VSWA mapping + active = getattr(self, '_vswa_active_pool_id', None) + if pool_id == active and not self.is_cuda_graph: + return # Buffer already has the right data + n = self._paged_kv_indices.numel() if self.is_cuda_graph else ( + self.num_generation_blocks + self.num_context_blocks) + src = self._vswa_pool_indices_cache[pool_id][:n] + self._paged_kv_indices[:n].copy_(src, non_blocking=True) + self._vswa_active_pool_id = pool_id + # Keep the host mirror in lockstep with the device buffer so decode + # plans build their block tables from the active pool's indices. + self._host_paged_kv_indices = self._host_pool_indices[pool_id] - # Build only the live (or previously live) rectangle on the host. This preserves - # padded-row zeroing when batch size shrinks without launching full-capacity - # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. - host_table = host_block_tables[:update_rows, :update_width] - host_table.zero_() - host_pool_indices = self._host_pool_indices[pool_id] - # Pool indices are flattened as context blocks followed by each generation request's - # blocks. - source_offset = self.num_context_blocks - # This loop scales with the number of generation requests. Vectorizing ragged - # rows would require padded mask or index tensor, so keep contiguous row copies - # until profiling identifies this as a CPU bottleneck. - for row, num_blocks_for_row in enumerate(num_blocks_per_row): - num_blocks_for_row = int(num_blocks_for_row) - copy_width = min(num_blocks_for_row, table_width) - host_table[row, :copy_width].copy_( - host_pool_indices[source_offset:source_offset + - copy_width]) - # Advance by the uncropped count so the next request starts at the correct - # offset even if this row hit `table_width`. - source_offset += num_blocks_for_row - # Keep the graph-captured device address stable and transfer only the compact - # rectangle prepared in pinned host memory. - block_tables[:update_rows, :update_width].copy_( - host_block_tables[:update_rows, :update_width], - non_blocking=True) - wrappers.decode_block_table_active_rows = rows - wrappers.decode_block_table_active_width = active_width - kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) - if kv_lens_buf is not None: - decode_kv_lens = _to_int32_tensor( - kv_lens_host[self.num_contexts:self.num_contexts + - self.num_generations]) - kv_lens_buf[:self.num_generations].copy_(decode_kv_lens, - non_blocking=True) - if self.num_generations < batch_size: - kv_lens_buf[self.num_generations:batch_size].zero_() + @property + def paged_kv_last_page_len(self) -> torch.Tensor: + return self._paged_kv_last_page_len[:self.num_contexts + + self.num_generations] - # Refresh captured FA2 schedules only after all page metadata updates. - # Under CUDA graphs every non-trtllm-gen decode wrapper carries - # fa2_plan_num_blocks, so this is what keeps replayed plans current; - # the deferral below only affects wrappers without a recorded tuple. - # Defer ordinary multi-wrapper plans to forward_impl; single-wrapper - # models still plan eagerly because forward_impl cannot plan during - # graph capture. - active_wrappers = [ - pp for pp in self._plan_params_to_wrappers - if pp.attention_mask_data is None - ] - defer_plan = len(active_wrappers) > 1 - if not (self._is_separate_kv_draft_view and self.is_cuda_graph): - self._refresh_fa2_cuda_graph_plans() - self._clean_cached_plans(defer_plan=defer_plan) + @property + def qo_indptr(self) -> torch.Tensor: + return self._qo_indptr[:self.num_contexts + self.num_generations + 1] - if (not self._is_shared_kv_draft_view - and not self._is_separate_kv_draft_view - and self._draft_metadata is not None): - self._draft_metadata.core()._sync_draft_view(md) + @property + def kv_indptr(self) -> torch.Tensor: + return self._kv_indptr[:self.num_contexts + self.num_generations + 1] - if md.has_cross_sub_metadata: - md.refresh_cross_query_side().prepare() + @property + def cached_token_lens(self) -> torch.Tensor: + return self._cached_token_lens[:self.num_contexts + + self.num_generations] - def needs_plan(self, plan_params: PlanParams) -> bool: - if plan_params not in self._plan_params_to_wrappers: - return True + @property + def batch_indices(self) -> torch.Tensor: + return self._batch_indices[:self.num_tokens] - wrappers = self._plan_params_to_wrappers[plan_params] - return not wrappers.is_planned + @property + def positions(self) -> torch.Tensor: + return self._positions[:self.num_tokens] - def get_prefill_wrapper( - self, plan_params: PlanParams - ) -> flashinfer.BatchPrefillWithPagedKVCacheWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[plan_params].prefill_wrapper - assert result is not None, "Prefill wrapper was not created in plan()" - return result + def resync_length_derived_state(self) -> None: + """Refresh a separate-KV draft view after a speculative sub-step. - def get_decode_wrapper( - self, plan_params: PlanParams - ) -> flashinfer.BatchDecodeWithPagedKVCacheWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[plan_params].decode_wrapper - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - if result._backend != "trtllm-gen": - raise ValueError( - "FlashInfer draft metadata views require the trtllm-gen " - "decode backend.") - self._publish_decode_wrapper_kv_lens(result) - return result + FlashInfer replans from the lengths on its next forward, so only a + draft view over a separate KV cache has state to refresh here. + """ + if not self._is_separate_kv_draft_view: + return + self._prepare_full_draft_page_table() + self._update_draft_kv_lengths() - def get_ragged_prefill_wrapper( - self, plan_params: PlanParams) -> _RaggedPrefillWrapper: - assert plan_params in self._plan_params_to_wrappers, "Plan params not found, make sure to call plan()" - result = self._plan_params_to_wrappers[ - plan_params].ragged_prefill_wrapper - assert result is not None, "Ragged prefill wrapper was not created in plan()" - return result + def cuda_graph_copy_changes(self, max_batch_size: int, + max_draft_tokens: int) -> Dict[str, Any]: + # The graph's wrappers and buffers are sized for the graph's batch. + return { + "max_num_requests": max_batch_size, + "max_num_tokens": max_batch_size * (1 + max_draft_tokens), + } - def plan_ragged( + def get_draft_metadata( self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - plan_params: RaggedPlanParams, - ) -> flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper: - """Plan MLA ragged prefill with expanded K, V (not paged).""" - if self._ragged_prefill_wrapper is None: - self._ragged_prefill_wrapper = flashinfer.prefill.BatchPrefillWithRaggedKVCacheWrapper( - self.workspace_buffer, - "NHD", - ) - if self._mla_ragged_plan_params != plan_params: - self._mla_ragged_planned = False - self._mla_ragged_plan_params = plan_params - - if self._mla_ragged_planned: - return self._ragged_prefill_wrapper + draft_kv_cache_manager: Optional[Any] = None, + ) -> "FlashInferAttentionMetadata": + """Return a planned draft view over shared or separate KV cache.""" + md = self.metadata + uses_shared_kv_cache = draft_kv_cache_manager is None + if self._draft_metadata is None: + # A metadata of its own over the same configuration: a shared-KV + # view plans one query token per request over the target's cache; + # a separate-KV view addresses the draft manager's pools. Both run + # on the target's workspace. + if uses_shared_kv_cache: + draft_metadata = md.constructed_copy( + cuda_graph_buffers=None, + max_num_tokens=self.max_num_requests, + seq_lens=torch.ones((self.max_num_requests, ), + dtype=torch.int), + num_contexts=0, + seq_lens_kv=None, + cross=None, + ) + else: + draft_metadata = md.constructed_copy( + share=False, + cuda_graph_buffers=None, + kv_cache_manager=draft_kv_cache_manager, + workspace_buffer=self.workspace_buffer, + seq_lens=self.seq_lens, + num_contexts=self.num_contexts, + seq_lens_kv=None, + cross=None, + ) + # The view's core is this class's, so this class sets it up. + draft_core = draft_metadata.core() + draft_core._is_shared_kv_draft_view = uses_shared_kv_cache + draft_core._is_separate_kv_draft_view = not uses_shared_kv_cache + if not uses_shared_kv_cache: + # Keep the existing speculative worker's backend-neutral + # kv_lens_cuda protocol scoped to the separate-KV draft view. + draft_core.kv_lens_cuda = draft_core._draft_kv_runtime_lens + self._draft_metadata = draft_metadata + draft_metadata.core()._sync_draft_view(md) + elif (self._draft_metadata.core()._is_shared_kv_draft_view + != uses_shared_kv_cache): + raise RuntimeError( + "FlashInfer metadata cannot mix shared and separate draft KV " + "views.") + if not uses_shared_kv_cache: + # The separate view's staging copy of the draft manager holds the + # same tokens as the target's: take the lengths the overlap + # scheduler corrected on the device since prepare. + self._draft_metadata.core()._follow_staging_lengths( + self.mixed_precision_staging) + return self._draft_metadata - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA ragged prefill while the stream " - "is capturing. Make sure prepare() has run at least one " - "warmup forward pass before capture.") + def _follow_staging_lengths( + self, source: Optional["MixedPrecisionKVStaging"]) -> None: + """Copy ``source``'s staging lengths into this view's staging copy.""" + view = self.mixed_precision_staging + if (view is not None and source is not None and view.lengths is not None + and source.lengths is not None): + view.lengths.copy_(source.lengths) - # Split append_paged_mla_kv_cache from plan() when this wrapper needs a - # new plan. Reusing a cached plan avoids this sync on later layers. - torch.cuda.current_stream().synchronize() + def _mixed_precision_staging_name(self) -> str: + # A separate-KV draft view stages the draft manager. + if self._is_separate_kv_draft_view: + return "draft_mixed" + return super()._mixed_precision_staging_name() - self._do_plan_ragged(qo_indptr, kv_indptr, plan_params) - self._mla_ragged_planned = True - - return self._ragged_prefill_wrapper - - def _do_plan_ragged( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - plan_params: RaggedPlanParams, - ) -> None: - assert self._ragged_prefill_wrapper is not None - self._ragged_prefill_wrapper.plan( - qo_indptr, - kv_indptr, - plan_params.num_heads, - plan_params.num_kv_heads, - plan_params.head_dim, - head_dim_vo=plan_params.head_dim_vo, - use_fp16_qk_reduction=False, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, - ) + def _sync_draft_view(self, target: "FlashInferAttentionMetadata") -> None: + """Refresh a shared- or separate-KV draft metadata view.""" + md = self.metadata + if not (self._is_shared_kv_draft_view + or self._is_separate_kv_draft_view): + raise RuntimeError("Only a draft metadata view can be synchronized") - def plan_mla_decode( - self, - plan_params: MLAPlanParams, - ) -> object: - """Plan MLA decode using BatchMLAPagedAttentionWrapper. + num_seqs = target.num_seqs + self.request_ids = target.request_ids + self.prompt_lens = target.prompt_lens + self.kv_cache_params = target.kv_cache_params + self.all_rank_num_tokens = target.all_rank_num_tokens - Caches the wrapper and plan params; the actual plan() call is driven - by prepare() so it runs outside of CUDA graph capture. - """ - if self._mla_decode_wrapper is None: - self._mla_decode_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( - self.workspace_buffer, - use_cuda_graph=self.is_cuda_graph, - qo_indptr=self._mla_qo_indptr_buf, - kv_indptr=self.paged_kv_indptr_decode, - kv_indices=self._paged_kv_indices, - kv_len_arr=self._mla_kv_len_arr_buf, - backend="auto", + if self._is_separate_kv_draft_view: + self.padded_num_tokens = target.padded_num_tokens + self.seq_lens = target.seq_lens + self.seq_lens_kv = None + self.num_contexts = target.num_contexts + self._uses_full_draft_page_table = False + md.prepare() + torch.add( + target.core()._cached_token_lens[:num_seqs], + target.seq_lens_kv_cuda[:num_seqs], + out=self._draft_kv_runtime_lens[:num_seqs], ) + if self.num_contexts == 0: + self._prepare_full_draft_page_table() + return - if self._mla_decode_plan_params != plan_params: - self._mla_decode_planned = False + self.kv_cache_manager = target.kv_cache_manager + self.seq_lens = torch.ones((num_seqs, ), dtype=torch.int) + self.seq_lens_kv = None + self.num_contexts = 0 + self.num_blocks = list(target.num_blocks) + self.num_context_blocks = 0 + self.num_generation_blocks = sum(self.num_blocks) + self.num_ctx_cached_tokens = 0 + self._multi_item_params = None - # Cache params so prepare() can re-plan on subsequent forward passes. - self._mla_decode_plan_params = plan_params + total_blocks = self.num_generation_blocks + self._paged_kv_indices[:total_blocks].copy_( + target.core()._paged_kv_indices[:total_blocks], non_blocking=True) + self._host_paged_kv_indices = target.core()._host_paged_kv_indices - if self._mla_decode_planned: - return self._mla_decode_wrapper + if (target.core()._vswa_layer_to_pool is not None + and target.core()._vswa_pool_indices_cache is not None): + self._vswa_pool_indices_cache = {} + self._host_pool_indices = dict(target.core()._host_pool_indices) + for pool_id, source in target.core()._vswa_pool_indices_cache.items( + ): + destination = self._vswa_pool_bufs[pool_id] + destination[:total_blocks].copy_(source[:total_blocks], + non_blocking=True) + self._vswa_pool_indices_cache[pool_id] = destination + primary_pool_id = self._primary_kv_pool_id + self._vswa_active_pool_id = primary_pool_id + self._host_paged_kv_indices = self._host_pool_indices[ + primary_pool_id] + self._paged_kv_indices[:total_blocks].copy_( + self._vswa_pool_indices_cache[primary_pool_id][:total_blocks], + non_blocking=True) + else: + self._vswa_pool_indices_cache = None + self._host_pool_indices = {} + self._vswa_active_pool_id = None - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA decode while the stream is " - "capturing. Make sure prepare() has run at least one warmup " - "forward pass before capture.") + host_indptr = maybe_pin_memory( + torch.from_numpy( + np.concatenate([[0], np.cumsum(self.num_blocks) + ]).astype(np.int32, copy=False))) + self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, + non_blocking=True) + self._host_paged_kv_indptr_decode = host_indptr + self.paged_kv_indptr_prefill[0].zero_() + self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] + self._paged_kv_last_page_len[:num_seqs].copy_( + target.core()._paged_kv_last_page_len[:num_seqs], non_blocking=True) + self._cached_token_lens[:num_seqs].copy_( + target.core()._cached_token_lens[:num_seqs], non_blocking=True) - # Split append_paged_mla_kv_cache from plan() on a cache miss. prepare() - # calls _do_plan_mla_decode() directly and does not need this sync. - torch.cuda.current_stream().synchronize() - self._do_plan_mla_decode(plan_params) - self._mla_decode_planned = True - return self._mla_decode_wrapper + host_qo_indptr = torch.arange(num_seqs + 1, + dtype=torch.int32, + pin_memory=prefer_pinned()) + self._qo_indptr[:num_seqs + 1].copy_(host_qo_indptr, non_blocking=True) + full_kv_lens = (target.core()._cached_token_lens[:num_seqs] + + target.seq_lens_kv_cuda[:num_seqs]) + self._draft_kv_runtime_lens[:num_seqs].copy_(full_kv_lens) - def plan_mla_context( + if self.is_cuda_graph: + # Graph replay keeps the captured trtllm-gen plan. Refresh only + # its stable block-table buffers; re-planning the wrapper would + # mutate state owned by the captured kernel. + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + self._build_decode_block_tables(plan_params, wrappers) + else: + self._clean_cached_plans(defer_plan=False) + + def update_shared_kv_draft_lengths( self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - kv_last_page_len: torch.Tensor, - plan_params: MLAPlanParams, - ) -> object: - """Plan MLA context with cached KV using BatchMLAPagedAttentionWrapper.""" - if self._mla_context_wrapper is None: - self._mla_context_wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper( - self.workspace_buffer, - use_cuda_graph=False, - backend="auto", - ) + target: "FlashInferAttentionMetadata", + num_accepted_tokens: torch.Tensor, + num_contexts: int, + ) -> None: + """Publish the accepted target prefix to assistant decode wrappers.""" + if not self._is_shared_kv_draft_view: + raise RuntimeError( + "Draft KV lengths can only be set on the shared-KV view") + num_seqs = target.num_seqs + cached_lens = target.core()._cached_token_lens[:num_seqs] + runtime_lens = self._draft_kv_runtime_lens[:num_seqs] + if num_contexts > 0: + runtime_lens[:num_contexts].copy_( + cached_lens[:num_contexts] + + target.seq_lens_kv_cuda[:num_contexts]) + runtime_lens[num_contexts:num_seqs].copy_( + cached_lens[num_contexts:num_seqs] + + num_accepted_tokens[num_contexts:num_seqs]) + self._update_draft_kv_lengths() - if self._mla_context_plan_params != plan_params: - self._mla_context_planned = False - self._mla_context_plan_params = plan_params + def apply_spec_decode_kv_lens_offsets( + self, + offsets: torch.Tensor, + num_generations: int, + tokens_per_generation: int, + *, + num_chunked_contexts: int = 0, + restore: bool = False, + ) -> None: + """Apply overlap-scheduler corrections to FlashInfer's live KV state.""" + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + raise RuntimeError( + "Speculative KV offsets must be applied to target metadata") + if not self._uses_full_generation_page_table: + return + if num_chunked_contexts == 0 and num_generations == 0: + return - if self._mla_context_planned: - return self._mla_context_wrapper + direction = -1 if restore else 1 + num_contexts = self.num_contexts + if num_chunked_contexts > 0: + row_slice = slice(num_contexts - num_chunked_contexts, num_contexts) + runtime_offsets = offsets[:num_chunked_contexts] + num_runtime_tokens = num_chunked_contexts * tokens_per_generation + token_slice = slice(self.num_ctx_tokens - num_runtime_tokens, + self.num_ctx_tokens) + else: + row_slice = slice(num_contexts, num_contexts + num_generations) + runtime_offsets = offsets[:num_generations] + num_runtime_tokens = num_generations * tokens_per_generation + token_slice = slice(self.num_ctx_tokens, + self.num_ctx_tokens + num_runtime_tokens) - if self.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise ValueError( - "Cannot plan() flashinfer MLA context while the stream is " - "capturing. Make sure prepare() has run at least one warmup " - "forward pass before capture.") + self._cached_token_lens[row_slice].add_(runtime_offsets, + alpha=direction) + self._logical_kv_lens[row_slice].add_(runtime_offsets, alpha=direction) + token_offsets = runtime_offsets.repeat_interleave(tokens_per_generation) + self._positions[token_slice].add_(token_offsets, alpha=direction) + view = self.mixed_precision_staging + if view is not None and view.manager is self.kv_cache_manager: + # Keep the staging lengths in step with the KV lengths. + view.lengths[:, row_slice].add_(runtime_offsets, alpha=direction) - # Split append_paged_mla_kv_cache from plan() when this wrapper needs a - # new plan. Reusing a cached plan avoids this sync on later layers. - torch.cuda.current_stream().synchronize() + for wrappers in self._plan_params_to_wrappers.values(): + self._publish_decode_wrapper_kv_lens(wrappers.decode_wrapper) - self._do_plan_mla_context( - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_len, - plan_params, - ) - self._mla_context_planned = True + def _publish_decode_wrapper_kv_lens(self, decode_wrapper) -> None: + """Publish device-logical lengths to a trtllm-gen decode wrapper.""" + kv_lens_buffer = getattr(decode_wrapper, "_kv_lens_buffer", None) + if kv_lens_buffer is None or self.num_generations == 0: + return - return self._mla_context_wrapper + if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: + kv_lens_buffer[:self.num_generations].copy_( + self._draft_kv_runtime_lens[:self.num_generations]) + return + if not self._uses_full_generation_page_table: + return - def _do_plan_mla_context( - self, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, - kv_indices: torch.Tensor, - kv_last_page_len: torch.Tensor, - plan_params: MLAPlanParams, - ) -> None: - assert self._mla_context_wrapper is not None - - num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] - kv_len_arr = (num_pages_per_seq - - 1) * plan_params.page_size + kv_last_page_len - - self._mla_context_wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - kv_len_arr, - plan_params.num_heads, - plan_params.kv_lora_rank, - plan_params.qk_rope_head_dim, - plan_params.page_size, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, - ) - - def _do_plan_mla_decode(self, plan_params: MLAPlanParams) -> None: - """Compute MLA decode plan inputs and call wrapper.plan(). - - Must run outside of CUDA graph capture. - """ - num_gen = self.num_generations - kv_indptr = self.paged_kv_indptr_decode[:num_gen + 1] - kv_indices = self._paged_kv_indices[self.num_context_blocks:self. - num_context_blocks + - self.num_generation_blocks] - kv_last_page = self._paged_kv_last_page_len[self.num_contexts:self. - num_contexts + num_gen] - - # _qo_indptr is ordered [context_seqs..., generation_seqs...]; rebase - # the generation slice to 0. - num_ctx = self.num_contexts - qo_indptr = self._qo_indptr[num_ctx:num_ctx + num_gen + - 1] - self._qo_indptr[num_ctx] - - if self._uses_full_generation_page_table: - kv_len_arr = self._logical_kv_lens[num_ctx:num_ctx + num_gen] - else: - num_pages_per_seq = kv_indptr[1:] - kv_indptr[:-1] - kv_len_arr = (num_pages_per_seq - - 1) * plan_params.page_size + kv_last_page - - self._mla_decode_wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - kv_len_arr, - plan_params.num_heads, - plan_params.kv_lora_rank, - plan_params.qk_rope_head_dim, - plan_params.page_size, - causal=True, - q_data_type=plan_params.q_dtype, - kv_data_type=plan_params.kv_dtype, - sm_scale=plan_params.sm_scale, + start = self.num_contexts + end = start + self.num_generations + torch.add( + self._cached_token_lens[start:end], + self.seq_lens_kv_cuda[start:end], + out=kv_lens_buffer[:self.num_generations], ) - @property - def paged_kv_indices(self) -> torch.Tensor: - return self._paged_kv_indices[:self.num_generation_blocks + - self.num_context_blocks] + def _debug_validate_page_table(self, kv_lens_host, logical_num_blocks, + num_blocks) -> None: + """Diagnostic: check the page table handed to FlashInfer is well formed. - def get_paged_kv_indices_for_layer(self, layer_idx: int) -> torch.Tensor: - """Return page indices for the pool that *layer_idx* belongs to. + Every structural claim the kernel relies on is already on the host by + the time this runs -- the index mirrors, the indptr host copies and the + length arrays are all numpy or CPU tensors -- so the check costs no GPU + work and no synchronization, which is what makes it usable on a path + where a synchronization can hide the fault being chased. - For non-VSWA models this returns the default shared indices. - For VSWA models it returns the pool-specific indices. + Off unless ``TRTLLM_FI_PAGE_TABLE_CHECK`` is set. Reports are bounded, + because a malformed row repeats on every step of the request that owns + it, and one line is printed when the check is armed and finds nothing, + so that a clean run is distinguishable from a run the environment + variable never reached. """ - if (self._vswa_layer_to_pool is None - or self._vswa_pool_indices_cache is None): - return self.paged_kv_indices - pool_id = self._vswa_layer_to_pool.get(layer_idx) - if pool_id is None: - return self.paged_kv_indices - total_blocks = self.num_generation_blocks + self.num_context_blocks - return self._vswa_pool_indices_cache[pool_id][:total_blocks] - - def _sanitize_swa_page_indices(self, page_indices: torch.Tensor, - layer_idx: int) -> None: - """Replace evicted SWA pages with a safe in-range page index.""" - window_vec = getattr(self.kv_cache_manager, 'max_attention_window_vec', - None) - if not window_vec: + if not os.environ.get(_PAGE_TABLE_CHECK_ENV, "").strip(): return - local_layer_idx = self.kv_cache_manager.layer_offsets[layer_idx] - if window_vec[local_layer_idx] is None: + cls = FlashInferAttentionMetadata + if cls._page_table_check_budget <= 0: return + pool_size = getattr(self.kv_cache_manager, 'blocks_in_primary_pool', + None) + pools = getattr(self, '_host_pool_indices', None) or { + 0: getattr(self, '_host_paged_kv_indices', None) + } + host_pools = { + pool_id: + (indices.cpu().numpy() if torch.is_tensor(indices) else indices) + for pool_id, indices in pools.items() + } + try: + problems = check_page_table( + host_pools, + num_blocks, + logical_num_blocks, + kv_lens_host, + self.page_size, + pool_size=pool_size, + ) + except Exception as exc: # pragma: no cover - a diagnostic must not break a run + cls._page_table_check_budget -= 1 + logger.warning( + f"[flashinfer] page table check failed to run: {exc}") + return + if problems: + cls._page_table_check_budget -= 1 + logger.warning( + f"[flashinfer] page table check, {self.num_contexts} context / " + f"{self.num_generations} generation rows: " + + "; ".join(problems)) + elif cls._page_table_check_budget == _PAGE_TABLE_CHECK_REPORTS: + cls._page_table_check_budget -= 1 + logger.info( + f"[flashinfer] page table check armed and clean on a batch of " + f"{self.num_contexts} context / {self.num_generations} " + f"generation rows, pool size {pool_size}") - # KVCacheManagerV2 marks evicted out-of-window pages with -1. - # FlashInfer may dereference page IDs before applying window_left, so - # keep masked positions in range. The SWA mask excludes their values - # from the attention result. - # - # Page 0 is a live page owned by whichever request holds it, so parking - # there has the kernel dereference a stranger's keys and values, with - # the mask as the only thing keeping them out of the result. When the - # KV cache manager has reserved a guard page (TRTLLM_KV_GUARD_PAGE), - # park on that instead: a page no request can own, holding a known - # pattern. The default -- no guard page -- keeps page 0. - page_indices.masked_fill_(page_indices == BAD_PAGE_INDEX, - self._swa_park_page(layer_idx)) - - def _swa_park_page(self, layer_idx: int) -> int: - """Page index that masked-out entries are pointed at for this layer.""" - guard = getattr(self.kv_cache_manager, 'guard_page_index', None) - if guard is None: - return 0 - page = guard(layer_idx) - return 0 if page is None else int(page) + def _prepare_full_draft_page_table(self) -> None: + """Expose every allocated draft page and use device KV lengths.""" + if self._uses_full_draft_page_table: + return + assert self.request_ids is not None + block_ids_per_seq = self.kv_cache_manager.get_batch_cache_indices( + self.request_ids, layer_idx=self._primary_kv_layer_idx) + self.num_blocks = [len(block_ids) for block_ids in block_ids_per_seq] + self.num_context_blocks = 0 + self.num_generation_blocks = sum(self.num_blocks) + paged_kv_indices = self.kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, + self.num_blocks, + layer_idx=self._primary_kv_layer_idx) + self._paged_kv_indices[:paged_kv_indices.numel()].copy_( + paged_kv_indices, non_blocking=True) + self._host_paged_kv_indices = paged_kv_indices - def swap_paged_kv_indices_for_layer(self, layer_idx: int) -> None: - """Copy pool-specific page indices into the shared buffer. + host_indptr = maybe_pin_memory( + torch.from_numpy( + np.concatenate([[0], np.cumsum(self.num_blocks) + ]).astype(np.int32, copy=False))) + num_seqs = self.num_seqs + self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, + non_blocking=True) + self._host_paged_kv_indptr_decode = host_indptr + self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] + self._uses_full_draft_page_table = True + self._update_draft_kv_lengths() - The FlashInfer wrappers reference ``_paged_kv_indices`` directly, - so we overwrite its contents with the correct pool's data before - each layer's plan/run cycle. We track the currently active pool - so we only copy when the pool actually changes. + if self.is_cuda_graph: + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + self._build_decode_block_tables(plan_params, wrappers) + elif not torch.cuda.is_current_stream_capturing(): + self._clean_cached_plans(defer_plan=False) - """ - if self._vswa_layer_to_pool is None: + def _update_draft_kv_lengths(self) -> None: + """Publish runtime KV lengths to a shared or separate draft view.""" + num_seqs = self.num_seqs + runtime_lens = self._draft_kv_runtime_lens[:num_seqs] + self._cached_token_lens[:num_seqs].copy_(runtime_lens) + self._paged_kv_last_page_len[:num_seqs].copy_(runtime_lens) + self._paged_kv_last_page_len[:num_seqs].sub_(1) + torch.remainder(self._paged_kv_last_page_len[:num_seqs], + self.page_size, + out=self._paged_kv_last_page_len[:num_seqs]) + self._paged_kv_last_page_len[:num_seqs].add_(1) + if self._is_shared_kv_draft_view: return - pool_id = self._vswa_layer_to_pool.get(layer_idx) - if pool_id is None: - return # Layer not in VSWA mapping - active = getattr(self, '_vswa_active_pool_id', None) - if pool_id == active and not self.is_cuda_graph: - return # Buffer already has the right data - n = self._paged_kv_indices.numel() if self.is_cuda_graph else ( - self.num_generation_blocks + self.num_context_blocks) - src = self._vswa_pool_indices_cache[pool_id][:n] - self._paged_kv_indices[:n].copy_(src, non_blocking=True) - self._vswa_active_pool_id = pool_id - # Keep the host mirror in lockstep with the device buffer so decode - # plans build their block tables from the active pool's indices. - self._host_paged_kv_indices = self._host_pool_indices[pool_id] - - @property - def paged_kv_last_page_len(self) -> torch.Tensor: - return self._paged_kv_last_page_len[:self.num_contexts + - self.num_generations] - @property - def qo_indptr(self) -> torch.Tensor: - return self._qo_indptr[:self.num_contexts + self.num_generations + 1] + self._cached_token_lens[:num_seqs].sub_(1) + self._positions[:num_seqs].copy_(self._cached_token_lens[:num_seqs]) + torch.arange(num_seqs + 1, + dtype=torch.int32, + device=self._qo_indptr.device, + out=self._qo_indptr[:num_seqs + 1]) + torch.arange(num_seqs, + dtype=torch.int32, + device=self._batch_indices.device, + out=self._batch_indices[:num_seqs]) - @property - def kv_indptr(self) -> torch.Tensor: - return self._kv_indptr[:self.num_contexts + self.num_generations + 1] + def __init__(self, metadata, *, source=None, **config) -> None: + super().__init__(metadata, source=source, **config) + # MLA wrappers and stable buffers. + # Cached plan params + is-planned flag let prepare() refresh the plan + # outside stream capture (flashinfer plan() does device->host syncs). + self._ragged_prefill_wrapper = None + self._mla_decode_wrapper = None + self._mla_context_wrapper = None + self._mla_ragged_plan_params = None + self._mla_context_plan_params = None + self._mla_decode_plan_params = None + self.num_ctx_cached_tokens = 0 + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False + self._mla_qo_indptr_buf = None + # Graph-stable page-index buffer of each VSWA pool, keyed by pool ID. CUDA + # graph copies share their source's buffers; a separate-KV draft view + # allocates its own. + self._vswa_pool_bufs = self.shared_or("_vswa_pool_bufs", lambda: dict()) + self._mla_kv_len_arr_buf = None + self._multi_item_params = None + self._draft_metadata = None + self._is_shared_kv_draft_view = False + self._is_separate_kv_draft_view = False + self._uses_full_draft_page_table = False + # Whether target generation rows expose reserved speculative pages while keeping their + # attention-visible lengths in `_logical_kv_lens`. + self._uses_full_generation_page_table = False + self.workspace_buffer = self.shared_or("workspace_buffer", + lambda: self.workspace_buffer) + buffers = metadata.cuda_graph_buffers + md = self.metadata + capture_graph = self.is_cuda_graph - @property - def cached_token_lens(self) -> torch.Tensor: - return self._cached_token_lens[:self.num_contexts + - self.num_generations] + if self.workspace_buffer is None: + # Note: even though flashinfer only recommends 128 MB, we have to push it + # a bit higher to cover all possible CUDA graph cases. If it's too small, + # warmup will crash. + # FlashInfer's split-KV decode plan sizes its temp buffers proportionally to + # SM count; 320 MB was tuned for a 148-SM baseline, so scale up (never down) + # on GPUs with more SMs. + baseline_sms = 148 + baseline_bytes = 320 * 1024 * 1024 + num_sms = torch.cuda.get_device_properties( + torch.cuda.current_device()).multi_processor_count + workspace_bytes = max(baseline_bytes, + baseline_bytes * num_sms // baseline_sms) + self.workspace_buffer = md.get_empty( + buffers, + (workspace_bytes, ), + dtype=torch.uint8, + cache_name="workspace_buffer", + capture_graph=capture_graph, + ) - @property - def batch_indices(self) -> torch.Tensor: - return self._batch_indices[:self.num_tokens] + self.paged_kv_indptr_decode = torch.empty((self.max_num_requests + 1, ), + device='cuda', + dtype=torch.int) + self.paged_kv_indptr_prefill = torch.empty( + (self.max_num_requests + 1, ), device='cuda', dtype=torch.int) + self._paged_kv_last_page_len = torch.empty((self.max_num_requests, ), + device='cuda', + dtype=torch.int) - @property - def positions(self) -> torch.Tensor: - return self._positions[:self.num_tokens] + self._qo_indptr = torch.zeros(self.max_num_requests + 1, + device='cuda', + dtype=torch.int) - def resync_length_derived_state(self) -> None: - """Refresh a separate-KV draft view after a speculative sub-step. + self._kv_indptr = torch.zeros( + self.max_num_requests + 1, device='cuda', + dtype=torch.int) if self.is_cross else self._qo_indptr - FlashInfer replans from the lengths on its next forward, so only a - draft view over a separate KV cache has state to refresh here. - """ - if not self._is_separate_kv_draft_view: - return - self._prepare_full_draft_page_table() - self._update_draft_kv_lengths() + self._ragged_qo_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_ragged_qo_indptr_buf", + capture_graph=capture_graph, + ) + self._ragged_kv_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_ragged_kv_indptr_buf", + capture_graph=capture_graph, + ) - def cuda_graph_copy_changes(self, max_batch_size: int, - max_draft_tokens: int) -> Dict[str, Any]: - # The graph's wrappers and buffers are sized for the graph's batch. - return { - "max_num_requests": max_batch_size, - "max_num_tokens": max_batch_size * (1 + max_draft_tokens), - } + self._cached_token_lens = torch.empty((self.max_num_requests, ), + dtype=torch.int, + device='cuda') + self._logical_kv_lens = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int, + cache_name="_logical_kv_lens", + capture_graph=capture_graph, + ) + self._draft_kv_runtime_lens = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int, + cache_name="_draft_kv_runtime_lens", + capture_graph=capture_graph, + ) + self._batch_indices = torch.empty((self.max_num_tokens, ), + dtype=torch.int, + device='cuda') + self._positions = torch.empty((self.max_num_tokens, ), + dtype=torch.int, + device='cuda') + self._plan_params_to_wrappers = {} - def get_draft_metadata( - self, - draft_kv_cache_manager: Optional[Any] = None, - ) -> "FlashInferAttentionMetadata": - """Return a planned draft view over shared or separate KV cache.""" - md = self.metadata - uses_shared_kv_cache = draft_kv_cache_manager is None - if self._draft_metadata is None: - # A metadata of its own over the same configuration: a shared-KV - # view plans one query token per request over the target's cache; - # a separate-KV view addresses the draft manager's pools. Both run - # on the target's workspace. - if uses_shared_kv_cache: - draft_metadata = md.constructed_copy( - cuda_graph_buffers=None, - max_num_tokens=self.max_num_requests, - seq_lens=torch.ones((self.max_num_requests, ), - dtype=torch.int), - num_contexts=0, - seq_lens_kv=None, - cross=None, - ) - else: - draft_metadata = md.constructed_copy( - share=False, - cuda_graph_buffers=None, - kv_cache_manager=draft_kv_cache_manager, - workspace_buffer=self.workspace_buffer, - seq_lens=self.seq_lens, - num_contexts=self.num_contexts, - seq_lens_kv=None, - cross=None, - ) - # The view's core is this class's, so this class sets it up. - draft_core = draft_metadata.core() - draft_core._is_shared_kv_draft_view = uses_shared_kv_cache - draft_core._is_separate_kv_draft_view = not uses_shared_kv_cache - if not uses_shared_kv_cache: - # Keep the existing speculative worker's backend-neutral - # kv_lens_cuda protocol scoped to the separate-KV draft view. - draft_core.kv_lens_cuda = draft_core._draft_kv_runtime_lens - self._draft_metadata = draft_metadata - draft_metadata.core()._sync_draft_view(md) - elif (self._draft_metadata.core()._is_shared_kv_draft_view - != uses_shared_kv_cache): - raise RuntimeError( - "FlashInfer metadata cannot mix shared and separate draft KV " - "views.") - if not uses_shared_kv_cache: - # The separate view's staging copy of the draft manager holds the - # same tokens as the target's: take the lengths the overlap - # scheduler corrected on the device since prepare. - self._draft_metadata.core()._follow_staging_lengths( - self.mixed_precision_staging) - return self._draft_metadata + # Host-side state for sync-free trtllm-gen decode plans, retained by + # prepare(): a host mirror of _paged_kv_indices (kept in lockstep + # with the device buffer, with per-pool copies for VSWA swaps) and a + # host copy of the decode indptr. The per-plan_params persistent + # block-table buffers live on FlashInferWrappers. + self._host_pool_indices: Dict[int, torch.Tensor] = {} + self._host_paged_kv_indices: Optional[torch.Tensor] = None + self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None + self._uses_full_generation_page_table = False + self._max_num_blocks_per_seq = 0 - def _follow_staging_lengths( - self, source: Optional["MixedPrecisionKVStaging"]) -> None: - """Copy ``source``'s staging lengths into this view's staging copy.""" - view = self.mixed_precision_staging - if (view is not None and source is not None and view.lengths is not None - and source.lengths is not None): - view.lengths.copy_(source.lengths) + # VSWA (Variable Sliding Window Attention): models with per-layer + # max_attention_window create separate V2 pool groups with independent + # page numbering. We need per-pool paged_kv_indices so each layer can + # use the indices that match its pool's buffer. + self._vswa_layer_to_pool: Optional[Dict[int, int]] = None + self._vswa_pool_indices_cache: Optional[Dict[int, torch.Tensor]] = None + self._primary_kv_layer_idx: Optional[int] = None + self._primary_kv_pool_id = 0 - def _mixed_precision_staging_name(self) -> str: - # A separate-KV draft view stages the draft manager. - if self._is_separate_kv_draft_view: - return "draft_mixed" - return super()._mixed_precision_staging_name() + if self.kv_cache_manager is not None: + blocks_in_primary_pool = self.kv_cache_manager.blocks_in_primary_pool + attention_layer_indices = _get_attention_layer_indices( + self.kv_cache_manager) - def _sync_draft_view(self, target: "FlashInferAttentionMetadata") -> None: - """Refresh a shared- or separate-KV draft metadata view.""" - md = self.metadata - if not (self._is_shared_kv_draft_view - or self._is_separate_kv_draft_view): - raise RuntimeError("Only a draft metadata view can be synchronized") + # Maximum block count across attention pools: sizes the shared + # page-index buffer and the VSWA pool buffers below. + max_num_blocks = blocks_in_primary_pool + for layer_idx in attention_layer_indices: + layer_buffer = self.kv_cache_manager.get_buffers(layer_idx) + if layer_buffer is not None: + max_num_blocks = max(max_num_blocks, layer_buffer.shape[0]) + if self._has_mixed_precision_staging(): + # The staging page space spans every sequence slot. + max_num_blocks = max( + max_num_blocks, + (self.max_num_sequences or self.max_num_requests) * + self.kv_cache_manager.max_blocks_per_seq) + self._paged_kv_indices = md.get_empty( + buffers, + (max_num_blocks, ), + dtype=torch.int, + cache_name="_paged_kv_indices", + capture_graph=capture_graph, + ) + self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq - num_seqs = target.num_seqs - self.request_ids = target.request_ids - self.prompt_lens = target.prompt_lens - self.kv_cache_params = target.kv_cache_params - self.all_rank_num_tokens = target.all_rank_num_tokens - - if self._is_separate_kv_draft_view: - self.padded_num_tokens = target.padded_num_tokens - self.seq_lens = target.seq_lens - self.seq_lens_kv = None - self.num_contexts = target.num_contexts - self._uses_full_draft_page_table = False - md.prepare() - torch.add( - target.core()._cached_token_lens[:num_seqs], - target.seq_lens_kv_cuda[:num_seqs], - out=self._draft_kv_runtime_lens[:num_seqs], - ) - if self.num_contexts == 0: - self._prepare_full_draft_page_table() - return - - self.kv_cache_manager = target.kv_cache_manager - self.seq_lens = torch.ones((num_seqs, ), dtype=torch.int) - self.seq_lens_kv = None - self.num_contexts = 0 - self.num_blocks = list(target.num_blocks) - self.num_context_blocks = 0 - self.num_generation_blocks = sum(self.num_blocks) - self.num_ctx_cached_tokens = 0 + # Layers may share one page-index list only when they are in the + # same pool, page-index scale, and KV factor: VSWA splits pools, + # and per-layer geometry (e.g. Gemma4 sliding/global head_dim) + # splits scales even when all windows collapse to max_seq_len and + # is_vswa is False. Mixed K-only/KV layers can use different view + # indices even with the same physical scale. Guarded on V2-specific + # attributes (V1 managers lack the per-pool infrastructure). + mgr = self.kv_cache_manager + get_scale = getattr(mgr, 'get_layer_page_index_scale', None) + get_kv_factor = getattr(mgr, 'get_layer_kv_factor', None) + layer_space: Dict[int, int] = {} + if hasattr(mgr, 'layer_to_pool_mapping_dict') and get_scale: + space_ids = {} + for layer_idx in attention_layer_indices: + layer_offset = mgr.layer_offsets[layer_idx] + key = (mgr.layer_to_pool_mapping_dict[layer_offset], + get_scale(layer_idx), + get_kv_factor(layer_idx) if get_kv_factor else None) + if (self._has_mixed_precision_staging() and layer_offset + in mgr.mixed_precision_high_layers): + key = ("mixed_precision_staging", ) + space_ids.setdefault(key, len(space_ids)) + layer_space[layer_idx] = space_ids[key] + if layer_space: + self._primary_kv_layer_idx = next(iter(layer_space)) + self._primary_kv_pool_id = layer_space[ + self._primary_kv_layer_idx] + if layer_space and (getattr(mgr, 'is_vswa', False) + or len(set(layer_space.values())) > 1): + self._vswa_layer_to_pool = {} + self._vswa_pool_to_rep_layer: Dict[int, int] = {} + for layer_idx, pool_id in layer_space.items(): + self._vswa_layer_to_pool[layer_idx] = pool_id + if pool_id not in self._vswa_pool_to_rep_layer: + self._vswa_pool_to_rep_layer[pool_id] = layer_idx + # Pre-allocate VSWA pool cache buffers. These must be + # stable (never reallocated) so that CUDA-graph-recorded + # copies reference valid addresses across replays. + # max_num_blocks (computed above) covers ALL pools so that + # secondary pool buffers are large enough. + for pool_id in set(self._vswa_layer_to_pool.values()): + if self._vswa_pool_bufs.get(pool_id) is None: + self._vswa_pool_bufs[pool_id] = torch.empty( + max_num_blocks, dtype=torch.int, device='cuda') + # Stable buffers for FlashInfer MLA decode; required for CUDA graphs. + self._mla_qo_indptr_buf = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + dtype=torch.int32, + cache_name="_mla_qo_indptr_buf", + capture_graph=capture_graph, + ) + self._mla_kv_len_arr_buf = md.get_empty( + buffers, + (self.max_num_requests, ), + dtype=torch.int32, + cache_name="_mla_kv_len_arr_buf", + capture_graph=capture_graph, + ) + # Rebind the wrapper to the freshly allocated buffers. + self._ragged_prefill_wrapper = None + self._mla_decode_wrapper = None + self._mla_context_wrapper = None + self._mla_ragged_plan_params = None + self._mla_context_plan_params = None + self._mla_decode_plan_params = None + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False self._multi_item_params = None - total_blocks = self.num_generation_blocks - self._paged_kv_indices[:total_blocks].copy_( - target.core()._paged_kv_indices[:total_blocks], non_blocking=True) - self._host_paged_kv_indices = target.core()._host_paged_kv_indices - - if (target.core()._vswa_layer_to_pool is not None - and target.core()._vswa_pool_indices_cache is not None): - self._vswa_pool_indices_cache = {} - self._host_pool_indices = dict(target.core()._host_pool_indices) - for pool_id, source in target.core()._vswa_pool_indices_cache.items( - ): - destination = self._vswa_pool_bufs[pool_id] - destination[:total_blocks].copy_(source[:total_blocks], - non_blocking=True) - self._vswa_pool_indices_cache[pool_id] = destination - primary_pool_id = self._primary_kv_pool_id - self._vswa_active_pool_id = primary_pool_id - self._host_paged_kv_indices = self._host_pool_indices[ - primary_pool_id] - self._paged_kv_indices[:total_blocks].copy_( - self._vswa_pool_indices_cache[primary_pool_id][:total_blocks], - non_blocking=True) - else: - self._vswa_pool_indices_cache = None - self._host_pool_indices = {} - self._vswa_active_pool_id = None - - host_indptr = maybe_pin_memory( - torch.from_numpy( - np.concatenate([[0], np.cumsum(self.num_blocks) - ]).astype(np.int32, copy=False))) - self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, - non_blocking=True) - self._host_paged_kv_indptr_decode = host_indptr - self.paged_kv_indptr_prefill[0].zero_() - self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] - self._paged_kv_last_page_len[:num_seqs].copy_( - target.core()._paged_kv_last_page_len[:num_seqs], non_blocking=True) - self._cached_token_lens[:num_seqs].copy_( - target.core()._cached_token_lens[:num_seqs], non_blocking=True) - - host_qo_indptr = torch.arange(num_seqs + 1, - dtype=torch.int32, - pin_memory=prefer_pinned()) - self._qo_indptr[:num_seqs + 1].copy_(host_qo_indptr, non_blocking=True) - full_kv_lens = (target.core()._cached_token_lens[:num_seqs] + - target.seq_lens_kv_cuda[:num_seqs]) - self._draft_kv_runtime_lens[:num_seqs].copy_(full_kv_lens) - - if self.is_cuda_graph: - # Graph replay keeps the captured trtllm-gen plan. Refresh only - # its stable block-table buffers; re-planning the wrapper would - # mutate state owned by the captured kernel. - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - self._build_decode_block_tables(plan_params, wrappers) - else: - self._clean_cached_plans(defer_plan=False) + @property + def page_size(self) -> int: + """ + Number of tokens per cache page + """ + assert self.kv_cache_manager is not None, ( + "page_size is undefined without a KV cache manager; use the " + "ragged prefill path instead.") + return self.kv_cache_manager.tokens_per_block - def update_shared_kv_draft_lengths( + def _plan_ragged_no_kv( self, - target: "FlashInferAttentionMetadata", - num_accepted_tokens: torch.Tensor, - num_contexts: int, + plan_params: PlanParams, + ragged_prefill_wrapper: Any, + flashinfer_backend: str, ) -> None: - """Publish the accepted target prefix to assistant decode wrappers.""" - if not self._is_shared_kv_draft_view: - raise RuntimeError( - "Draft KV lengths can only be set on the shared-KV view") - num_seqs = target.num_seqs - cached_lens = target.core()._cached_token_lens[:num_seqs] - runtime_lens = self._draft_kv_runtime_lens[:num_seqs] - if num_contexts > 0: - runtime_lens[:num_contexts].copy_( - cached_lens[:num_contexts] + - target.seq_lens_kv_cuda[:num_contexts]) - runtime_lens[num_contexts:num_seqs].copy_( - cached_lens[num_contexts:num_seqs] + - num_accepted_tokens[num_contexts:num_seqs]) - self._update_draft_kv_lengths() + is_causal = plan_params.attention_mask_type == AttentionMaskType.causal + if plan_params.attention_mask_data is not None: + window_left = -1 + else: + window_left = plan_params.window_left - def apply_spec_decode_kv_lens_offsets( - self, - offsets: torch.Tensor, - num_generations: int, - tokens_per_generation: int, - *, - num_chunked_contexts: int = 0, - restore: bool = False, - ) -> None: - """Apply overlap-scheduler corrections to FlashInfer's live KV state.""" - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - raise RuntimeError( - "Speculative KV offsets must be applied to target metadata") - if not self._uses_full_generation_page_table: - return - if num_chunked_contexts == 0 and num_generations == 0: - return + # Lengths are already on GPU via AttentionMetadata (seq_lens setter -> _seq_lens_cuda). + assert self.seq_lens_cuda is not None + assert self.seq_lens is not None - direction = -1 if restore else 1 + # NB: When kv_cache_manager is None (e.g. ViT), ragged prefill runs only for the context phase. + # Restrict seq_lens and index tensors to the first num_contexts entries accordingly. + # + # NB: flashinfer expects seq_len (BatchPrefillWithPagedKVCacheWrapper only), paged_kv_last_page_len, qo_indptr, + # and paged_kv_indptr to be host tensors (if not, it synchronously copies them to the host). + # => not using self.qo_indptr and self.kv_indptr here num_contexts = self.num_contexts - if num_chunked_contexts > 0: - row_slice = slice(num_contexts - num_chunked_contexts, num_contexts) - runtime_offsets = offsets[:num_chunked_contexts] - num_runtime_tokens = num_chunked_contexts * tokens_per_generation - token_slice = slice(self.num_ctx_tokens - num_runtime_tokens, - self.num_ctx_tokens) - else: - row_slice = slice(num_contexts, num_contexts + num_generations) - runtime_offsets = offsets[:num_generations] - num_runtime_tokens = num_generations * tokens_per_generation - token_slice = slice(self.num_ctx_tokens, - self.num_ctx_tokens + num_runtime_tokens) - - self._cached_token_lens[row_slice].add_(runtime_offsets, - alpha=direction) - self._logical_kv_lens[row_slice].add_(runtime_offsets, alpha=direction) - token_offsets = runtime_offsets.repeat_interleave(tokens_per_generation) - self._positions[token_slice].add_(token_offsets, alpha=direction) - view = self.mixed_precision_staging - if view is not None and view.manager is self.kv_cache_manager: - # Keep the staging lengths in step with the KV lengths. - view.lengths[:, row_slice].add_(runtime_offsets, alpha=direction) + q_and_kv_seqlens = self.seq_lens[:num_contexts] + q_and_kv_seqlens_cuda = self.seq_lens_cuda[:self.num_contexts] + query_output_indptr = torch.zeros( + num_contexts + 1, + dtype=torch.int32, + pin_memory=prefer_pinned(), + ) + key_value_indptr = query_output_indptr + num_query_output_heads = plan_params.num_heads + num_key_value_heads = plan_params.num_kv_heads + attention_head_dim = plan_params.head_dim - for wrappers in self._plan_params_to_wrappers.values(): - self._publish_decode_wrapper_kv_lens(wrappers.decode_wrapper) - - def _publish_decode_wrapper_kv_lens(self, decode_wrapper) -> None: - """Publish device-logical lengths to a trtllm-gen decode wrapper.""" - kv_lens_buffer = getattr(decode_wrapper, "_kv_lens_buffer", None) - if kv_lens_buffer is None or self.num_generations == 0: - return - - if self._is_shared_kv_draft_view or self._is_separate_kv_draft_view: - kv_lens_buffer[:self.num_generations].copy_( - self._draft_kv_runtime_lens[:self.num_generations]) - return - if not self._uses_full_generation_page_table: - return - - start = self.num_contexts - end = start + self.num_generations - torch.add( - self._cached_token_lens[start:end], - self.seq_lens_kv_cuda[start:end], - out=kv_lens_buffer[:self.num_generations], - ) - - def _debug_validate_page_table(self, kv_lens_host, logical_num_blocks, - num_blocks) -> None: - """Diagnostic: check the page table handed to FlashInfer is well formed. - - Every structural claim the kernel relies on is already on the host by - the time this runs -- the index mirrors, the indptr host copies and the - length arrays are all numpy or CPU tensors -- so the check costs no GPU - work and no synchronization, which is what makes it usable on a path - where a synchronization can hide the fault being chased. - - Off unless ``TRTLLM_FI_PAGE_TABLE_CHECK`` is set. Reports are bounded, - because a malformed row repeats on every step of the request that owns - it, and one line is printed when the check is armed and finds nothing, - so that a clean run is distinguishable from a run the environment - variable never reached. - """ - if not os.environ.get(_PAGE_TABLE_CHECK_ENV, "").strip(): - return - cls = FlashInferAttentionMetadata - if cls._page_table_check_budget <= 0: - return - pool_size = getattr(self.kv_cache_manager, 'blocks_in_primary_pool', - None) - pools = getattr(self, '_host_pool_indices', None) or { - 0: getattr(self, '_host_paged_kv_indices', None) - } - host_pools = { - pool_id: - (indices.cpu().numpy() if torch.is_tensor(indices) else indices) - for pool_id, indices in pools.items() - } - try: - problems = check_page_table( - host_pools, - num_blocks, - logical_num_blocks, - kv_lens_host, - self.page_size, - pool_size=pool_size, - ) - except Exception as exc: # pragma: no cover - a diagnostic must not break a run - cls._page_table_check_budget -= 1 - logger.warning( - f"[flashinfer] page table check failed to run: {exc}") - return - if problems: - cls._page_table_check_budget -= 1 - logger.warning( - f"[flashinfer] page table check, {self.num_contexts} context / " - f"{self.num_generations} generation rows: " + - "; ".join(problems)) - elif cls._page_table_check_budget == _PAGE_TABLE_CHECK_REPORTS: - cls._page_table_check_budget -= 1 - logger.info( - f"[flashinfer] page table check armed and clean on a batch of " - f"{self.num_contexts} context / {self.num_generations} " - f"generation rows, pool size {pool_size}") - - def _prepare_full_draft_page_table(self) -> None: - """Expose every allocated draft page and use device KV lengths.""" - if self._uses_full_draft_page_table: - return - assert self.request_ids is not None - block_ids_per_seq = self.kv_cache_manager.get_batch_cache_indices( - self.request_ids, layer_idx=self._primary_kv_layer_idx) - self.num_blocks = [len(block_ids) for block_ids in block_ids_per_seq] - self.num_context_blocks = 0 - self.num_generation_blocks = sum(self.num_blocks) - paged_kv_indices = self.kv_cache_manager.get_batch_cache_indices_flat( - self.request_ids, - self.num_blocks, - layer_idx=self._primary_kv_layer_idx) - self._paged_kv_indices[:paged_kv_indices.numel()].copy_( - paged_kv_indices, non_blocking=True) - self._host_paged_kv_indices = paged_kv_indices - - host_indptr = maybe_pin_memory( - torch.from_numpy( - np.concatenate([[0], np.cumsum(self.num_blocks) - ]).astype(np.int32, copy=False))) - num_seqs = self.num_seqs - self.paged_kv_indptr_decode[:num_seqs + 1].copy_(host_indptr, - non_blocking=True) - self._host_paged_kv_indptr_decode = host_indptr - self.paged_kv_indptr = self.paged_kv_indptr_decode[:num_seqs + 1] - self._uses_full_draft_page_table = True - self._update_draft_kv_lengths() - - if self.is_cuda_graph: - for plan_params, wrappers in self._plan_params_to_wrappers.items(): - self._build_decode_block_tables(plan_params, wrappers) - elif not torch.cuda.is_current_stream_capturing(): - self._clean_cached_plans(defer_plan=False) - - def _update_draft_kv_lengths(self) -> None: - """Publish runtime KV lengths to a shared or separate draft view.""" - num_seqs = self.num_seqs - runtime_lens = self._draft_kv_runtime_lens[:num_seqs] - self._cached_token_lens[:num_seqs].copy_(runtime_lens) - self._paged_kv_last_page_len[:num_seqs].copy_(runtime_lens) - self._paged_kv_last_page_len[:num_seqs].sub_(1) - torch.remainder(self._paged_kv_last_page_len[:num_seqs], - self.page_size, - out=self._paged_kv_last_page_len[:num_seqs]) - self._paged_kv_last_page_len[:num_seqs].add_(1) - if self._is_shared_kv_draft_view: - return - - self._cached_token_lens[:num_seqs].sub_(1) - self._positions[:num_seqs].copy_(self._cached_token_lens[:num_seqs]) - torch.arange(num_seqs + 1, - dtype=torch.int32, - device=self._qo_indptr.device, - out=self._qo_indptr[:num_seqs + 1]) - torch.arange(num_seqs, - dtype=torch.int32, - device=self._batch_indices.device, - out=self._batch_indices[:num_seqs]) - - @property - def page_size(self) -> int: - """ - Number of tokens per cache page - """ - assert self.kv_cache_manager is not None, ( - "page_size is undefined without a KV cache manager; use the " - "ragged prefill path instead.") - return self.kv_cache_manager.tokens_per_block - - def _plan_ragged_no_kv( - self, - plan_params: PlanParams, - ragged_prefill_wrapper: Any, - flashinfer_backend: str, - ) -> None: - is_causal = plan_params.attention_mask_type == AttentionMaskType.causal - if plan_params.attention_mask_data is not None: - window_left = -1 - else: - window_left = plan_params.window_left - - # Lengths are already on GPU via AttentionMetadata (seq_lens setter -> _seq_lens_cuda). - assert self.seq_lens_cuda is not None - assert self.seq_lens is not None - - # NB: When kv_cache_manager is None (e.g. ViT), ragged prefill runs only for the context phase. - # Restrict seq_lens and index tensors to the first num_contexts entries accordingly. - # - # NB: flashinfer expects seq_len (BatchPrefillWithPagedKVCacheWrapper only), paged_kv_last_page_len, qo_indptr, - # and paged_kv_indptr to be host tensors (if not, it synchronously copies them to the host). - # => not using self.qo_indptr and self.kv_indptr here - num_contexts = self.num_contexts - q_and_kv_seqlens = self.seq_lens[:num_contexts] - q_and_kv_seqlens_cuda = self.seq_lens_cuda[:self.num_contexts] - query_output_indptr = torch.zeros( - num_contexts + 1, - dtype=torch.int32, - pin_memory=prefer_pinned(), - ) - key_value_indptr = query_output_indptr - num_query_output_heads = plan_params.num_heads - num_key_value_heads = plan_params.num_kv_heads - attention_head_dim = plan_params.head_dim - - max_query_tokens_per_sequence = 0 - if num_contexts > 0: - query_output_indptr[1:].copy_(torch.cumsum(q_and_kv_seqlens, dim=0)) - # key_value_indptr is query_output_indptr - max_query_tokens_per_sequence = cast(int, - q_and_kv_seqlens.max().item()) + max_query_tokens_per_sequence = 0 + if num_contexts > 0: + query_output_indptr[1:].copy_(torch.cumsum(q_and_kv_seqlens, dim=0)) + # key_value_indptr is query_output_indptr + max_query_tokens_per_sequence = cast(int, + q_and_kv_seqlens.max().item()) if (multi_item_params := plan_params.multi_item_params) is not None: if flashinfer_backend not in ["fa2", "fa3"]: @@ -2043,6 +1649,400 @@ def _refresh_fa2_cuda_graph_plans(self) -> None: wrappers.is_planned = False self._plan_with_params(plan_params) + def derive(self) -> None: + md = self.metadata + + def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: + """Host int32 staging tensor for async H2D copies; pinned when + beneficial. + + torch.tensor(..., pin_memory=True) rejects numpy sources, hence + from_numpy + pin. + """ + return maybe_pin_memory( + torch.from_numpy(arr.astype(np.int32, copy=False))) + + extra_attrs = get_model_extra_attrs() + if extra_attrs is None: + get_global_attrs().attention_metadata = weakref.ref(md) + # start and end indices of each sequence in the ragged query + assert self.seq_lens_cuda is not None + torch.cumsum(self.seq_lens_cuda, + dim=0, + dtype=torch.int32, + out=self._qo_indptr[1:self.seq_lens_cuda.size(0) + 1]) + + if self.multi_item_part_lens is not None: + self._multi_item_params = self._process_multi_item_part_lens( + self.multi_item_part_lens, device=self.seq_lens_cuda.device) + else: + self._multi_item_params = None + + if self.kv_cache_manager is None: + assert self.request_ids is not None + assert self.num_generations == 0, ( + "FlashInfer without a KV cache manager only supports context-only " + "batches (num_generations == 0) in TRT-LLM.") + if self.is_cross: + raise NotImplementedError( + "FlashInfer without a KV cache manager is not tested for cross attention." + ) + self.kv_cache_params = KVCacheParams(use_cache=False) + n = self.num_seqs + self._cached_token_lens[:n].zero_() + self.num_ctx_cached_tokens = 0 + self._clean_cached_plans(defer_plan=False) + return + + if self._multi_item_params is not None: + raise ValueError( + "multi_item_part_lens with KV cache is not supported") + + # number of tokens in the kv cache for each sequence in the batch + num_cached_tokens_per_seq = self.kv_cache_params.num_cached_tokens_per_seq + cached_token_lens = torch.tensor(num_cached_tokens_per_seq, + dtype=torch.int, + pin_memory=prefer_pinned()) + self._cached_token_lens[:cached_token_lens.size(0)].copy_( + cached_token_lens, non_blocking=True) + if self.num_contexts > 0: + self.num_ctx_cached_tokens = sum( + num_cached_tokens_per_seq[:self.num_contexts]) + else: + self.num_ctx_cached_tokens = 0 + + # Number of tokens needed in the KV cache for each sequence after the next pass. + # Kept on the host: every consumer below needs host values, so a device-side computation + # would force a sync per step. These logical block counts describe committed history, not + # all pages that the KV manager has reserved for fixed-width speculative appends. + kv_lens_host = np.asarray(num_cached_tokens_per_seq, + dtype=np.int64) + self.seq_lens_kv.numpy() + logical_num_blocks = ((kv_lens_host + self.page_size - 1) // + self.page_size) + num_blocks = logical_num_blocks + use_full_generation_page_table = ( + self.kv_cache_params.use_full_generation_page_table) + self._uses_full_generation_page_table = use_full_generation_page_table + if use_full_generation_page_table: + # Generation rows need every reserved page to keep a speculative append addressable + # across a page boundary. Treating that wider table as the logical sequence would make + # FlashInfer attend to uncommitted draft slots, so plans and positions use the separate + # device logical lengths populated below. + assert self.request_ids is not None + num_blocks = np.asarray( + _get_page_table_num_blocks( + self.kv_cache_manager, + self.request_ids, + logical_num_blocks.tolist(), + self.num_contexts, + ), + dtype=np.int64, + ) + self.num_blocks = num_blocks.tolist() + + assert self.request_ids is not None + + # start and end indices of each sequence in the ragged key and value + # for md attention it's the same as qo_indptr so avoid computing twice. + if self.is_cross: + # start and end indices of each sequence in the ragged key and value + torch.cumsum(self.seq_lens_kv_cuda, + dim=0, + dtype=torch.int32, + out=self._kv_indptr[1:self.seq_lens_kv_cuda.size(0) + + 1]) + + # number of cache blocks used by each sequence in the cache + # NOTE: do not use len(block_ids) - that will give you a number + # that can be too big if using chunked prefill/kv cache reuse + # since we allocate all blocks ahead of time. + self.num_context_blocks = sum(self.num_blocks[:self.num_contexts]) + self.num_generation_blocks = sum(self.num_blocks[self.num_contexts:]) + + if self._has_mixed_precision_staging(): + self.prepare_mixed_precision_kv( + self.kv_layout, self.kv_cache_manager.tokens_per_block) + + # indices of used cache blocks for each sequence + primary_layer_idx = self._primary_kv_layer_idx + kv_cache_manager = (self.get_mixed_precision_kv_view(primary_layer_idx) + or self.kv_cache_manager) + paged_kv_indices = kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, self.num_blocks, layer_idx=primary_layer_idx) + if primary_layer_idx is not None: + self._sanitize_swa_page_indices(paged_kv_indices, primary_layer_idx) + + self._paged_kv_indices[:paged_kv_indices.size(0)].copy_( + paged_kv_indices, non_blocking=True) + + # Retain a host mirror of _paged_kv_indices: decode plans build the + # trtllm-gen block tables from host data, with no GPU round trips. + # The VSWA block below re-points the mirror at the active pool's + # copy whenever the device buffer is swapped. + self._host_paged_kv_indices = paged_kv_indices + + # VSWA: build per-pool page index CUDA tensors so each layer can use + # the indices that match its own pool's buffer. Tensors live on CUDA + # so that forward_impl swap via copy_() is device-to-device (CUDA-graph + # capturable). + if self._vswa_layer_to_pool is not None: + unique_pools = set(self._vswa_layer_to_pool.values()) + primary_pool_id = self._primary_kv_pool_id + # Use dedicated pre-allocated buffers for each pool's indices. + # These buffers are created in __post_init__ so their addresses + # stay stable across CUDA-graph replays. + total_idx = paged_kv_indices.size(0) + primary_buf = self._vswa_pool_bufs[primary_pool_id] + primary_buf[:total_idx].copy_(self._paged_kv_indices[:total_idx], + non_blocking=True) + self._vswa_pool_indices_cache = { + primary_pool_id: primary_buf, + } + self._host_pool_indices = {primary_pool_id: paged_kv_indices} + for pool_id in unique_pools: + if pool_id == primary_pool_id: + continue + rep_layer = self._vswa_pool_to_rep_layer[pool_id] + kv_cache_manager = (self.get_mixed_precision_kv_view(rep_layer) + or self.kv_cache_manager) + pool_indices = \ + kv_cache_manager.get_batch_cache_indices_flat( + self.request_ids, self.num_blocks, layer_idx=rep_layer) + self._sanitize_swa_page_indices(pool_indices, rep_layer) + buf = self._vswa_pool_bufs[pool_id] + buf[:pool_indices.size(0)].copy_(pool_indices, + non_blocking=True) + self._vswa_pool_indices_cache[pool_id] = buf + self._host_pool_indices[pool_id] = pool_indices + self._vswa_active_pool_id = primary_pool_id + + # number of tokens in the last cache block used by each sequence, + # derived from the logical rather than reservation-width page count. + paged_kv_last_page_len = _to_int32_tensor(kv_lens_host - + (logical_num_blocks - 1) * + self.page_size) + self._paged_kv_last_page_len[:paged_kv_last_page_len.size(0)].copy_( + paged_kv_last_page_len, non_blocking=True) + + # Ragged page table, see https://docs.flashinfer.ai/tutorials/kv_layout.html#page-table-layout + # For decoding, this MUST be allocated ahead of time (for CUDA graphs). + # Prefill is prepared here as well just for the sake of consistency. + paged_kv_indptr_decode = _to_int32_tensor( + np.concatenate([[0], np.cumsum(num_blocks[self.num_contexts:])])) + self.paged_kv_indptr_decode[:paged_kv_indptr_decode.size(0)].copy_( + paged_kv_indptr_decode, non_blocking=True) + # Retain the host copy: decode plans hand it to flashinfer so that + # its indptr.cpu()/get_seq_lens calls do no D2H work. + self._host_paged_kv_indptr_decode = paged_kv_indptr_decode + + paged_kv_indptr_prefill = _to_int32_tensor( + np.concatenate([[0], np.cumsum(num_blocks[:self.num_contexts])])) + self.paged_kv_indptr_prefill[:paged_kv_indptr_prefill.size(0)].copy_( + paged_kv_indptr_prefill, non_blocking=True) + + # This paged_kv_indptr attribute has both prefill and decode information in it. + # It's for the append_paged_kv_cache kernel. + if self.num_contexts == 0: + self.paged_kv_indptr = self.paged_kv_indptr_decode[: + paged_kv_indptr_decode + .size(0)] + elif self.num_generations == 0: + self.paged_kv_indptr = self.paged_kv_indptr_prefill[: + paged_kv_indptr_prefill + .size(0)] + else: + assert not self.is_cuda_graph, "Cannot mix decode/prefill with CUDA graphs" + # Accumulate on the host and stage through pinned memory: .cuda() + # on an unpinned tensor is a synchronous H2D that stalls the + # executor thread behind in-flight kernels on every mixed step. + self.paged_kv_indptr = _to_int32_tensor( + np.concatenate([[0], + np.cumsum(num_blocks)])).to(device='cuda', + non_blocking=True) + + # For cross attention, num_tokens is 0 during decode, and we don't need to update kv cache. + if self.num_tokens > 0: + if use_full_generation_page_table: + # The page-table indptr now describes addressable capacity. Deriving positions from + # it would count reserved pages as committed KV and potentially shift appends beyond + # the true sequence end. + logical_kv_lens = _to_int32_tensor(kv_lens_host) + self._logical_kv_lens[:logical_kv_lens.numel()].copy_( + logical_kv_lens, non_blocking=True) + position_kv_lens = self._logical_kv_lens[:self.num_seqs] + else: + position_kv_lens = flashinfer.get_seq_lens( + self.paged_kv_indptr, + self.paged_kv_last_page_len, + self.page_size, + ) + batch_indices, positions = flashinfer.get_batch_indices_positions( + self.kv_indptr, + position_kv_lens, + self.num_tokens, + ) + self._batch_indices[:batch_indices.size(0)].copy_(batch_indices, + non_blocking=True) + self._positions[:positions.size(0)].copy_(positions, + non_blocking=True) + + # Re-plan MLA wrappers outside of forward/capture using the params + # cached by prior warmup forwards. Forward still handles first-use or + # dtype/shape changes by syncing only on a plan cache miss. + self._mla_ragged_planned = False + self._mla_context_planned = False + self._mla_decode_planned = False + if (self.num_contexts > 0 and self._mla_ragged_plan_params is not None + and self._ragged_prefill_wrapper is not None): + ragged_indptr = self.qo_indptr[:self.num_contexts + 1] + self._do_plan_ragged(ragged_indptr, ragged_indptr, + self._mla_ragged_plan_params) + self._mla_ragged_planned = True + + if (self.num_contexts > 0 and self._mla_context_plan_params is not None + and self._mla_context_wrapper is not None): + num_contexts = self.num_contexts + num_context_blocks = self.num_context_blocks + context_qo_indptr = self.qo_indptr[:num_contexts + 1] + context_kv_indptr = self.paged_kv_indptr_prefill[:num_contexts + 1] + context_kv_indices = self._paged_kv_indices[:num_context_blocks] + context_last_page_len = self._paged_kv_last_page_len[:num_contexts] + self._do_plan_mla_context( + qo_indptr=context_qo_indptr, + kv_indptr=context_kv_indptr, + kv_indices=context_kv_indices, + kv_last_page_len=context_last_page_len, + plan_params=self._mla_context_plan_params, + ) + self._mla_context_planned = True + + # Re-plan the MLA decode wrapper outside of any stream capture. + if (self.num_generations > 0 + and self._mla_decode_plan_params is not None + and self._mla_decode_wrapper is not None): + self._do_plan_mla_decode(self._mla_decode_plan_params) + self._mla_decode_planned = True + + # VSWA: restore primary pool indices as the default. + if (self._vswa_layer_to_pool is not None + and self._vswa_pool_indices_cache is not None): + primary_pool_id = self._primary_kv_pool_id + total_blocks = self.num_generation_blocks + self.num_context_blocks + src = self._vswa_pool_indices_cache[primary_pool_id][:total_blocks] + self._paged_kv_indices[:total_blocks].copy_(src, non_blocking=True) + self._vswa_active_pool_id = primary_pool_id + self._host_paged_kv_indices = \ + self._host_pool_indices[primary_pool_id] + + self._debug_validate_page_table(kv_lens_host, logical_num_blocks, + num_blocks) + + # CUDA graph + trtllm-gen: update _block_tables and _kv_lens_buffer + # so the trtllm-gen decode kernel uses current page indices. + if (self.is_cuda_graph and self._vswa_layer_to_pool is not None + and self._vswa_pool_indices_cache is not None + and self.num_generations > 0): + decode_blocks = num_blocks[self.num_contexts:] + for plan_params, wrappers in self._plan_params_to_wrappers.items(): + if plan_params.attention_mask_data is not None: + continue + decode_wrapper = wrappers.decode_wrapper + block_tables = getattr(decode_wrapper, '_block_tables', None) + if block_tables is None: + continue + pool_id = plan_params.kv_pool_id + if pool_id is None: + continue + batch_size, table_width = block_tables.shape + rows = min(batch_size, self.num_generations) + num_blocks_per_row = decode_blocks[:rows] + active_width = min(int(num_blocks_per_row.max()), table_width) + # Include the previous live extent once when the batch or its longest sequence + # shrinks. The zero-filled copy below clears entries that are no longer live; the + # next update can shrink. + update_rows = min( + batch_size, + max(rows, wrappers.decode_block_table_active_rows)) + update_width = min( + table_width, + max(active_width, wrappers.decode_block_table_active_width)) + host_block_tables = wrappers.host_decode_block_tables + if (host_block_tables is None + or host_block_tables.size(0) < update_rows + or host_block_tables.size(1) < update_width): + # Grow geometrically to avoid reallocating whenever the longest sequence + # acquires one more KV-cache block. + host_width = min( + max(64, 1 << (update_width - 1).bit_length()), + table_width) + host_block_tables = torch.zeros( + (self.max_num_requests, host_width), + dtype=torch.int32, + pin_memory=prefer_pinned()) + wrappers.host_decode_block_tables = host_block_tables + + # Build only the live (or previously live) rectangle on the host. This preserves + # padded-row zeroing when batch size shrinks without launching full-capacity + # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. + host_table = host_block_tables[:update_rows, :update_width] + host_table.zero_() + host_pool_indices = self._host_pool_indices[pool_id] + # Pool indices are flattened as context blocks followed by each generation request's + # blocks. + source_offset = self.num_context_blocks + # This loop scales with the number of generation requests. Vectorizing ragged + # rows would require padded mask or index tensor, so keep contiguous row copies + # until profiling identifies this as a CPU bottleneck. + for row, num_blocks_for_row in enumerate(num_blocks_per_row): + num_blocks_for_row = int(num_blocks_for_row) + copy_width = min(num_blocks_for_row, table_width) + host_table[row, :copy_width].copy_( + host_pool_indices[source_offset:source_offset + + copy_width]) + # Advance by the uncropped count so the next request starts at the correct + # offset even if this row hit `table_width`. + source_offset += num_blocks_for_row + # Keep the graph-captured device address stable and transfer only the compact + # rectangle prepared in pinned host memory. + block_tables[:update_rows, :update_width].copy_( + host_block_tables[:update_rows, :update_width], + non_blocking=True) + wrappers.decode_block_table_active_rows = rows + wrappers.decode_block_table_active_width = active_width + kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) + if kv_lens_buf is not None: + decode_kv_lens = _to_int32_tensor( + kv_lens_host[self.num_contexts:self.num_contexts + + self.num_generations]) + kv_lens_buf[:self.num_generations].copy_(decode_kv_lens, + non_blocking=True) + if self.num_generations < batch_size: + kv_lens_buf[self.num_generations:batch_size].zero_() + + # Refresh captured FA2 schedules only after all page metadata updates. + # Under CUDA graphs every non-trtllm-gen decode wrapper carries + # fa2_plan_num_blocks, so this is what keeps replayed plans current; + # the deferral below only affects wrappers without a recorded tuple. + # Defer ordinary multi-wrapper plans to forward_impl; single-wrapper + # models still plan eagerly because forward_impl cannot plan during + # graph capture. + active_wrappers = [ + pp for pp in self._plan_params_to_wrappers + if pp.attention_mask_data is None + ] + defer_plan = len(active_wrappers) > 1 + if not (self._is_separate_kv_draft_view and self.is_cuda_graph): + self._refresh_fa2_cuda_graph_plans() + self._clean_cached_plans(defer_plan=defer_plan) + + if (not self._is_shared_kv_draft_view + and not self._is_separate_kv_draft_view + and self._draft_metadata is not None): + self._draft_metadata.core()._sync_draft_view(md) + + if md.has_cross_sub_metadata: + md.refresh_cross_query_side().prepare() + def plan(self, num_heads: int, num_kv_heads: int, diff --git a/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py b/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py index 1c882acda600..7197d59ccf9b 100644 --- a/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py +++ b/tensorrt_llm/_torch/attention/backends/fmha/fp4_mla.py @@ -38,7 +38,6 @@ ) from tensorrt_llm.bindings import DataType -from ..fp4_mla.state import find_fp4_mla from .interface import FmhaPhase from .phased import FmhaParams, PhasedFmha @@ -110,7 +109,7 @@ def _is_supported( ) ): raise NotImplementedError("FP4 MLA does not support sparse attention.") - state = find_fp4_mla(metadata) + state = metadata.fp4_mla_state if state is None or state.hp_pool is None: raise RuntimeError("FP4 MLA requires prepared high-precision KV state.") attention_input_type = forward_args.attention_input_type @@ -196,7 +195,7 @@ def run_mla_context(self, params: FmhaParams) -> None: raise RuntimeError("FP4 MLA context requires an output buffer.") if forward_args.latent_cache is None: raise RuntimeError("FP4 MLA context requires latent_cache.") - if find_fp4_mla(metadata).positions is None: + if metadata.fp4_mla_state.positions is None: raise RuntimeError("FP4 MLA context requires token positions.") if metadata.num_contexts <= 0: raise RuntimeError("FP4 MLA context requires context requests.") @@ -343,7 +342,7 @@ def forward_context_partition( def run_mla_generation(self, params: FmhaParams) -> None: attn = params.attn metadata = params.meta - fp4_mla_state = find_fp4_mla(metadata) + fp4_mla_state = metadata.fp4_mla_state forward_args = params.fwd q = params.query_input output = params.output diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py index cc88ea8f3898..fa8879296835 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_gather.py @@ -15,7 +15,6 @@ _validate_fp4_mla_kv_storage_shape, ) from .metadata import _materialize_fp4_mla_device_page_table_for_forward -from .state import find_fp4_mla def load_fp4_mla_chunked_kv_cache( @@ -31,7 +30,6 @@ def load_fp4_mla_chunked_kv_cache( qk_rope_head_dim: int, ) -> tuple[torch.Tensor, torch.Tensor]: """Gather one cached-prefix partition from dense FP4 MLA V2 storage.""" - fp4_mla_state = find_fp4_mla(metadata) if out_dtype not in (torch.float16, torch.bfloat16, torch.float32): raise TypeError(f"FP4 MLA chunk gather does not support output dtype {out_dtype}.") if num_ctx_cached_tokens < 0: @@ -89,11 +87,11 @@ def load_fp4_mla_chunked_kv_cache( if num_ctx_cached_tokens == 0: return compressed_kv, k_pe - if not bool(getattr(fp4_mla_state, "device_page_table", False)): + if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA chunk gather requires fixed-stride device page metadata.") _materialize_fp4_mla_device_page_table_for_forward(metadata) - page_table_stride = int(fp4_mla_state.page_table_stride) - page_ids = fp4_mla_state._paged_kv_indices + page_table_stride = int(metadata.fp4_mla_state.page_table_stride) + page_ids = metadata.fp4_mla_state._paged_kv_indices if ( page_table_stride <= 0 or not isinstance(page_ids, torch.Tensor) diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py index c84646efc5be..47c80e861731 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/cache_update.py @@ -56,7 +56,6 @@ _materialize_fp4_mla_device_page_table_for_forward, ) from .packed_plan import _fp4_mla_helix_plan_bounds, _fp4_mla_packed_plan_geometry -from .state import find_fp4_mla from .v_cache import ( _get_cutedsl_persistent_v_packed_cache, _get_fp4_mla_v_packed_pool_base, @@ -273,7 +272,6 @@ def _scatter_fp4_mla_kv_cache_2d_context( v_packed_base: Optional[torch.Tensor] = None, v_page_offset: int = 0, ) -> bool: - fp4_mla_state = find_fp4_mla(metadata) num_contexts = metadata.num_contexts if num_contexts > 0: prompt_lens_cpu = metadata.prompt_lens_cpu_runtime[:num_contexts] @@ -321,7 +319,7 @@ def _scatter_fp4_mla_kv_cache_2d_context( q_context_view = latent_cache q_head_blocks = 0 - hp_pool = getattr(fp4_mla_state, "hp_pool", None) + hp_pool = getattr(metadata.fp4_mla_state, "hp_pool", None) if not isinstance(hp_pool, torch.Tensor): raise TypeError("FP4 MLA high-precision KV pool must be a tensor.") if hp_pool.device != latent_cache.device: @@ -340,7 +338,7 @@ def _scatter_fp4_mla_kv_cache_2d_context( ) if hp_pool.stride(-1) != 1: raise ValueError("FP4 MLA high-precision KV pool must be contiguous in head_dim.") - hp_page_ids = fp4_mla_state.hp_page_indices + hp_page_ids = metadata.fp4_mla_state.hp_page_indices if not isinstance(hp_page_ids, torch.Tensor): raise RuntimeError("FP4 MLA context cache update requires HP page metadata.") store_hp_tail = num_contexts > 0 @@ -374,13 +372,13 @@ def _scatter_fp4_mla_kv_cache_2d_context( rotary_cos_sin_ptr, hp_pool, hp_page_ids, - fp4_mla_state.batch_indices, - fp4_mla_state.positions, - fp4_mla_state.paged_kv_indices, - fp4_mla_state.paged_kv_indptr, - fp4_mla_state.paged_kv_indices.shape[0], - fp4_mla_state.paged_kv_indptr.shape[0], - fp4_mla_state.batch_indices.shape[0], + metadata.fp4_mla_state.batch_indices, + metadata.fp4_mla_state.positions, + metadata.fp4_mla_state.paged_kv_indices, + metadata.fp4_mla_state.paged_kv_indptr, + metadata.fp4_mla_state.paged_kv_indices.shape[0], + metadata.fp4_mla_state.paged_kv_indptr.shape[0], + metadata.fp4_mla_state.batch_indices.shape[0], num_pages, num_layers, num_contexts, @@ -456,7 +454,6 @@ def _scatter_fp4_mla_kv_cache_2d_generation( helix_position_offsets: Optional[torch.Tensor], helix_is_inactive_rank: Optional[torch.Tensor], ) -> Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: - fp4_mla_state = find_fp4_mla(metadata) num_contexts = metadata.num_contexts num_seqs = metadata.num_seqs num_gen = num_seqs - num_contexts @@ -497,7 +494,7 @@ def _scatter_fp4_mla_kv_cache_2d_generation( ) _materialize_fp4_mla_device_page_table_for_forward(metadata, kv_lens_gen) - pool = getattr(fp4_mla_state, "hp_pool", None) + pool = getattr(metadata.fp4_mla_state, "hp_pool", None) if pool is None: raise RuntimeError("FP4 MLA 2D generation scatter requires the HP KV pool.") try: @@ -691,7 +688,7 @@ def launch_generation_update( helix_is_inactive_rank, page_ids, hp_page_ids, - find_fp4_mla(metadata).paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, page_ids_len, hp_page_ids.numel(), indptr_len, @@ -752,7 +749,7 @@ def launch_generation_update( launch_generation_update( launch_grid, page_ids_len=page_ids.shape[0], - indptr_len=fp4_mla_state.paged_kv_indptr_decode.shape[0], + indptr_len=metadata.fp4_mla_state.paged_kv_indptr_decode.shape[0], max_gen_tiles_variant=max(max_gen_tiles, 1), q_prefix_block_dim_variant=q_prefix_block_dim, q_prefix_blocks_variant=q_prefix_blocks, @@ -813,13 +810,12 @@ def scatter_fp4_mla_kv_cache( backend-ready residual FP4 Q. The return value reports whether the current phase updated the HP pool. """ - fp4_mla_state = find_fp4_mla(metadata) if phase == "generation": - fp4_mla_state.generation_cache_scattered = False - fp4_mla_state.prequantized_q = None - fp4_mla_state.prequantized_q_sf = None - fp4_mla_state.q_batch_capacity = None - fp4_mla_state.prequantized_q_rows_per_seq = None + metadata.fp4_mla_state.generation_cache_scattered = False + metadata.fp4_mla_state.prequantized_q = None + metadata.fp4_mla_state.prequantized_q_sf = None + metadata.fp4_mla_state.q_batch_capacity = None + metadata.fp4_mla_state.prequantized_q_rows_per_seq = None if latent_cache.numel() == 0: raise ValueError("FP4 MLA cache scatter requires at least one latent token.") if q_context is not None and not latent_cache.is_contiguous(): @@ -835,8 +831,8 @@ def scatter_fp4_mla_kv_cache( raise ValueError( f"FP4 MLA KV head_dim must be divisible by {FP4_BLOCK_SIZE}, got {head_dim}." ) - indices_len = fp4_mla_state.batch_indices.shape[0] - positions_len = fp4_mla_state.positions.shape[0] + indices_len = metadata.fp4_mla_state.batch_indices.shape[0] + positions_len = metadata.fp4_mla_state.positions.shape[0] if token_offset + num_tokens > indices_len or token_offset + num_tokens > positions_len: raise RuntimeError( f"FP4 MLA scatter would read batch_indices[{token_offset}:" @@ -868,7 +864,7 @@ def scatter_fp4_mla_kv_cache( context_only_cache = getattr(kv_cache_manager, "fp4_mla_context_only_cache", False) is True if context_only_cache and phase != "context": raise RuntimeError("FP4 MLA context-only cache cannot execute generation.") - if not context_only_cache and getattr(fp4_mla_state, "v_scale_pool", None) is None: + if not context_only_cache and getattr(metadata.fp4_mla_state, "v_scale_pool", None) is None: raise RuntimeError("FP4 MLA scatter requires the auxiliary V scale pool.") if metadata.page_size % FP4_BLOCK_SIZE != 0: raise ValueError( @@ -938,10 +934,10 @@ def scatter_fp4_mla_kv_cache( q_quant_input.device, rows_per_seq=q_rows_per_seq, ) - fp4_mla_state.prequantized_q = q_fp4_out - fp4_mla_state.prequantized_q_sf = q_sf_out - fp4_mla_state.q_batch_capacity = q_batch_capacity - fp4_mla_state.prequantized_q_rows_per_seq = q_rows_per_seq + metadata.fp4_mla_state.prequantized_q = q_fp4_out + metadata.fp4_mla_state.prequantized_q_sf = q_sf_out + metadata.fp4_mla_state.q_batch_capacity = q_batch_capacity + metadata.fp4_mla_state.prequantized_q_rows_per_seq = q_rows_per_seq hp_pool_updated = True cutedsl_backend = _fp4_mla_attention_backend() == _FP4_MLA_CUTEDSL_BACKEND fused_v_transpose = cutedsl_backend and _fp4_mla_cutedsl_fused_v_transpose_enabled() @@ -1035,7 +1031,7 @@ def scatter_fp4_mla_kv_cache( v_packed_base=v_packed_base, v_page_offset=v_page_offset, ) - v_pack_page_ids = fp4_mla_state.paged_kv_indices + v_pack_page_ids = metadata.fp4_mla_state.paged_kv_indices else: generation_state = _scatter_fp4_mla_kv_cache_2d_generation( metadata, @@ -1082,18 +1078,20 @@ def scatter_fp4_mla_kv_cache( ): if phase == "context": num_contexts = metadata.num_contexts - cutedsl_v_pack_page_ids = fp4_mla_state.paged_kv_indices[ - : fp4_mla_state.num_context_blocks + cutedsl_v_pack_page_ids = metadata.fp4_mla_state.paged_kv_indices[ + : metadata.fp4_mla_state.num_context_blocks ] - cutedsl_repack_page_indptr = fp4_mla_state.paged_kv_indptr[: num_contexts + 1] + cutedsl_repack_page_indptr = metadata.fp4_mla_state.paged_kv_indptr[: num_contexts + 1] cutedsl_repack_kv_lens = metadata.kv_lens_cuda_runtime[:num_contexts] cutedsl_repack_generation_lens = metadata.prompt_lens_cuda_runtime[:num_contexts] - cutedsl_repack_max_touched_pages = int(fp4_mla_state.context_repack_max_touched_pages) + cutedsl_repack_max_touched_pages = int( + metadata.fp4_mla_state.context_repack_max_touched_pages + ) elif generation_state is not None: kv_lens_gen, gen_lens_gen, generation_page_ids = generation_state num_gen = kv_lens_gen.numel() cutedsl_v_pack_page_ids = generation_page_ids - cutedsl_repack_page_indptr = fp4_mla_state.paged_kv_indptr_decode + cutedsl_repack_page_indptr = metadata.fp4_mla_state.paged_kv_indptr_decode cutedsl_repack_kv_lens = kv_lens_gen cutedsl_repack_generation_lens = gen_lens_gen cutedsl_repack_max_touched_pages = _ceil_div( @@ -1129,5 +1127,5 @@ def scatter_fp4_mla_kv_cache( num_valid_pages=v_pack_num_valid, ) if phase == "generation": - fp4_mla_state.generation_cache_scattered = hp_pool_updated + metadata.fp4_mla_state.generation_cache_scattered = hp_pool_updated return hp_pool_updated diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py index dc820ecc57b1..372a442026fa 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/decode.py @@ -55,7 +55,6 @@ _launch_fp4_mla_packed_plan, _PackedPlanGeometry, ) -from .state import find_fp4_mla from .v_cache import ( _get_cutedsl_persistent_v_packed_cache, _get_fp4_mla_v_packed_pool_base, @@ -167,7 +166,6 @@ def _run_triton_attention_decode( ``fp4_mla_triton.py``. Threads through the constexpr assume flags, TMA descriptors, occupancy/num-warps launch meta, and pipelined PV loop. """ - fp4_mla_state = find_fp4_mla(metadata) from .fp4_mla_triton import ( _fp4_mla_attention_group_reduce_stats_kernel as _attn_group_reduce_stats_kernel, ) @@ -314,7 +312,7 @@ def _tma_alloc(size: int, alignment: int, stream): global_scale, q_global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -379,8 +377,8 @@ def _tma_alloc(size: int, alignment: int, stream): # reduce during capture so we never allocate mid-capture. The single-level # reduce is numerically identical (it just launches fewer CTAs). if num_reduce_groups > 1 and torch.cuda.is_current_stream_capturing(): - gmax = fp4_mla_state.workspaces.get("_fp4_mla_attention_group_max_buf") - gsum = fp4_mla_state.workspaces.get("_fp4_mla_attention_group_sum_buf") + gmax = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_group_max_buf") + gsum = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_group_sum_buf") groups_ready = ( gmax is not None and gsum is not None @@ -460,7 +458,7 @@ def _tma_alloc(size: int, alignment: int, stream): max_scores, denom, page_max, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], max_scores.stride(0), @@ -530,7 +528,7 @@ def _tma_alloc(size: int, alignment: int, stream): # Fall back to the unsplit PV (numerically identical) during capture unless a # warmup forward already sized that workspace, so capture never allocates. if page_split > 1 and torch.cuda.is_current_stream_capturing(): - pbuf = fp4_mla_state.workspaces.get("_fp4_mla_attention_pv_partial_buf") + pbuf = metadata.fp4_mla_state.workspaces.get("_fp4_mla_attention_pv_partial_buf") partial_ready = ( pbuf is not None and pbuf.shape[0] >= num_queries @@ -560,7 +558,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -609,7 +607,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -683,7 +681,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -724,7 +722,7 @@ def _tma_alloc(size: int, alignment: int, stream): v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state.paged_kv_indptr_decode, kv_lens, src_page_ids.shape[0], kv_cache.shape[0], @@ -856,7 +854,7 @@ def _run_cutedsl_packed_decode( plan = plan_geometry.plan # The first FP4 MLA kernel of the step wrote the plan fused with the # generation lengths; a stale key means this call plans on other bounds. - if find_fp4_mla(metadata).packed_plan_key != plan_geometry.key: + if metadata.fp4_mla_state.packed_plan_key != plan_geometry.key: _launch_fp4_mla_packed_plan(plan_geometry, kv_lens, metadata) partial = _ensure_workspace_tensor( metadata, @@ -939,7 +937,6 @@ def run_fp4_mla_attention_decode( KV workspace is materialized on this path. Callers must supply the packed Q and scales produced by the fused generation cache update. """ - fp4_mla_state = find_fp4_mla(metadata) head_dim = kv_lora_rank + qk_rope_head_dim if qk_rope_head_dim != FP4_MLA_K_RESIDUAL_DIM: raise ValueError( @@ -1013,7 +1010,7 @@ def run_fp4_mla_attention_decode( "same-device int32 tensor covering every query token." ) helix_kv_bounds = helix_kv_bounds[:num_queries] - if getattr(fp4_mla_state, "v_scale_pool", None) is None: + if getattr(metadata.fp4_mla_state, "v_scale_pool", None) is None: raise RuntimeError( "FP4 MLA attention decode requires the auxiliary V scale pool to be allocated." ) @@ -1030,7 +1027,9 @@ def run_fp4_mla_attention_decode( capacity = int(q_batch_capacity) # The fused generation update records the Q layout it produced; metadata # that never went through it (contract tests) carries the legacy layout. - recorded_rows_per_seq = getattr(fp4_mla_state, "prequantized_q_rows_per_seq", None) + recorded_rows_per_seq = getattr( + getattr(metadata, "fp4_mla_state", None), "prequantized_q_rows_per_seq", None + ) packed_rows_per_seq = int(recorded_rows_per_seq or 0) if recorded_rows_per_seq is not None and packed_rows_per_seq != _fp4_mla_packed_rows_per_seq( num_heads, query_len_per_seq @@ -1090,7 +1089,7 @@ def run_fp4_mla_attention_decode( ) sf_cache = sf_cache.view(torch.float8_e4m3fn) - v_sf_pool = fp4_mla_state.v_scale_pool + v_sf_pool = metadata.fp4_mla_state.v_scale_pool v_sf = get_fp4_mla_v_scale_pool_view(metadata, v_head_dim=kv_lora_rank)[local_layer].view( torch.float8_e4m3fn ) @@ -1281,7 +1280,7 @@ def run_fp4_mla_attention_decode( global_scale, q_global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, output, num_heads=num_heads, @@ -1312,7 +1311,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, output, splits=select_split_count(max_pages * metadata.page_size), @@ -1364,7 +1363,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, kernel_output, splits=splits, @@ -1385,7 +1384,7 @@ def run_fp4_mla_attention_decode( core_v_sf, global_scale, src_page_ids, - fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], + metadata.fp4_mla_state.paged_kv_indptr_decode[: num_gen_seqs + 1], kv_lens, kernel_output, max_kv_len=max_pages * metadata.page_size, diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py index 8b2dbf01fa36..6cf2cbb012bc 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/fp4_mla_context.py @@ -25,7 +25,7 @@ from tensorrt_llm.quantization.mode import QuantMode from .fp4_mla_kernels import _fp8_mla_context_block_table_kernel -from .state import Fp4MlaState, find_fp4_mla +from .state import Fp4MlaState if TYPE_CHECKING: from tensorrt_llm._torch.attention.backends.trtllm import ( @@ -354,7 +354,7 @@ def _build_fp8_mla_context_metadata( host_total_kv_lens=scratch.host_total_kv_lens, kv_lens_cuda=kv_lens_cuda, kv_lens=kv_lens_cpu, - position_offsets=find_fp4_mla(meta).positions[: meta.num_ctx_tokens], + position_offsets=meta.fp4_mla_state.positions[: meta.num_ctx_tokens], without=(Fp4MlaState,), ) @@ -364,12 +364,11 @@ def _get_fp8_mla_context_metadata( scratch: _Fp8MlaContextScratch, ) -> "TrtllmAttentionMetadata": """Prepare and reuse one direct metadata view for all layers in a step.""" - fp4_mla_state = find_fp4_mla(meta) - state = fp4_mla_state._fp8_context_view + state = meta.fp4_mla_state._fp8_context_view if state is not None and state[0] is scratch: return state[1] scratch.prepare(meta) fp8_meta = _build_fp8_mla_context_metadata(meta, scratch) - fp4_mla_state._fp8_context_view = (scratch, fp8_meta) + meta.fp4_mla_state._fp8_context_view = (scratch, fp8_meta) return fp8_meta diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py index d8ef69317a39..e4bdf6e2c739 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/layout.py @@ -17,7 +17,6 @@ HP_BLOCK_SIZE, _ceil_div, ) -from .state import find_fp4_mla # FP4 MLA scale-layout helpers @@ -70,7 +69,7 @@ def get_fp4_mla_v_scale_pool_view( v_head_dim: int, ) -> torch.Tensor: """View the auxiliary MLA V-scale pool in Triton's block-scaled layout.""" - pool = getattr(find_fp4_mla(metadata), "v_scale_pool", None) + pool = getattr(metadata.fp4_mla_state, "v_scale_pool", None) if pool is None: raise RuntimeError("FP4 MLA V scale pool is not allocated.") @@ -101,7 +100,7 @@ def get_fp4_mla_v_scale_pool_view( def _get_fp4_mla_global_scale(metadata: Any, device: torch.device) -> torch.Tensor: - global_scale = getattr(find_fp4_mla(metadata), "kv_global_scale", None) + global_scale = getattr(metadata.fp4_mla_state, "kv_global_scale", None) if ( not isinstance(global_scale, torch.Tensor) or global_scale.device != device @@ -113,7 +112,7 @@ def _get_fp4_mla_global_scale(metadata: Any, device: torch.device) -> torch.Tens def _get_fp4_mla_q_global_scale(metadata: Any, device: torch.device) -> torch.Tensor: - global_scale = getattr(find_fp4_mla(metadata), "q_global_scale", None) + global_scale = getattr(metadata.fp4_mla_state, "q_global_scale", None) if ( not isinstance(global_scale, torch.Tensor) or global_scale.device != device @@ -236,7 +235,7 @@ def _ensure_workspace_tensor( dtype: torch.dtype, device: torch.device, ) -> torch.Tensor: - workspaces = find_fp4_mla(metadata).workspaces + workspaces = metadata.fp4_mla_state.workspaces tensor = workspaces.get(attr_name) needs_alloc = ( tensor is None diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py index f4866395613b..c248860aace5 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/metadata.py @@ -10,7 +10,6 @@ from .config import _FP4_MLA_PAGE_TABLE_TILE_SIZE, FP4_MLA_TOKENS_PER_BLOCK, _ceil_div from .packed_plan import _launch_fp4_mla_packed_plan, _PackedPlanGeometry -from .state import find_fp4_mla @triton.jit @@ -491,11 +490,10 @@ def configure_fp4_mla_device_page_table( table on the GPU. The materialization kernel decodes V2 page indices and refreshes rows from the final device KV lengths before cache update. """ - fp4_mla_state = find_fp4_mla(metadata) - fp4_mla_state.device_page_table = False - fp4_mla_state.device_page_table_valid = False - fp4_mla_state.page_table_stride = 0 - fp4_mla_state.context_repack_max_touched_pages = 1 + metadata.fp4_mla_state.device_page_table = False + metadata.fp4_mla_state.device_page_table_valid = False + metadata.fp4_mla_state.page_table_stride = 0 + metadata.fp4_mla_state.context_repack_max_touched_pages = 1 kv_cache_manager = getattr(metadata, "kv_cache_manager", None) num_contexts = int(getattr(metadata, "num_contexts", 0)) @@ -505,9 +503,9 @@ def configure_fp4_mla_device_page_table( num_context_tokens = int(getattr(metadata, "num_ctx_tokens", 0)) num_generation_tokens = num_tokens - num_context_tokens block_offsets = getattr(metadata, "kv_cache_block_offsets", None) - page_ids = getattr(fp4_mla_state, "_paged_kv_indices", None) - paged_kv_indptr = getattr(fp4_mla_state, "_paged_kv_indptr", None) - paged_kv_indptr_decode = getattr(fp4_mla_state, "paged_kv_indptr_decode", None) + page_ids = getattr(metadata.fp4_mla_state, "_paged_kv_indices", None) + paged_kv_indptr = getattr(metadata.fp4_mla_state, "_paged_kv_indptr", None) + paged_kv_indptr_decode = getattr(metadata.fp4_mla_state, "paged_kv_indptr_decode", None) max_page_capacity = int(getattr(kv_cache_manager, "max_blocks_per_seq", 0) or 0) page_spec = _fp4_mla_page_table_spec(kv_cache_manager) page_index_scale = int(page_spec.cache_page_index_scale) @@ -539,7 +537,7 @@ def configure_fp4_mla_device_page_table( if not supported: return False - hp_page_ids = getattr(fp4_mla_state, "hp_page_indices", None) + hp_page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) max_pool_id = max(page_spec.cache_pool_id, page_spec.hp_pool_id) if ( not isinstance(hp_page_ids, torch.Tensor) @@ -548,10 +546,10 @@ def configure_fp4_mla_device_page_table( or block_offsets.shape[0] <= max_pool_id ): return False - fp4_mla_state.cache_pool_id = int(page_spec.cache_pool_id) - fp4_mla_state.cache_page_index_scale = int(page_spec.cache_page_index_scale) - fp4_mla_state.hp_pool_id = int(page_spec.hp_pool_id) - fp4_mla_state.hp_page_index_scale = int(page_spec.hp_page_index_scale) + metadata.fp4_mla_state.cache_pool_id = int(page_spec.cache_pool_id) + metadata.fp4_mla_state.cache_page_index_scale = int(page_spec.cache_page_index_scale) + metadata.fp4_mla_state.hp_pool_id = int(page_spec.hp_pool_id) + metadata.fp4_mla_state.hp_page_index_scale = int(page_spec.hp_page_index_scale) max_pages = max_page_capacity host_kv_lens_available = ( @@ -583,7 +581,7 @@ def configure_fp4_mla_device_page_table( max_context_len, FP4_MLA_TOKENS_PER_BLOCK, ) - fp4_mla_state.context_repack_max_touched_pages = min( + metadata.fp4_mla_state.context_repack_max_touched_pages = min( max_pages, triton.next_power_of_2(max(1, max_context_pages)), ) @@ -608,15 +606,15 @@ def configure_fp4_mla_device_page_table( ) if not buffers_cover_table: return False - if fp4_mla_state.hp_page_indices.numel() < required_page_ids: + if metadata.fp4_mla_state.hp_page_indices.numel() < required_page_ids: return False - fp4_mla_state.device_page_table = True - fp4_mla_state.num_sequences = num_sequences - fp4_mla_state.page_table_stride = max_pages - fp4_mla_state.num_blocks = None - fp4_mla_state.num_context_blocks = num_contexts * max_pages - fp4_mla_state.num_generation_blocks = num_generation_sequences * max_pages + metadata.fp4_mla_state.device_page_table = True + metadata.fp4_mla_state.num_sequences = num_sequences + metadata.fp4_mla_state.page_table_stride = max_pages + metadata.fp4_mla_state.num_blocks = None + metadata.fp4_mla_state.num_context_blocks = num_contexts * max_pages + metadata.fp4_mla_state.num_generation_blocks = num_generation_sequences * max_pages return True @@ -626,16 +624,15 @@ def materialize_fp4_mla_device_page_table( generation_kv_lens: Optional[torch.Tensor] = None, ) -> None: """Refresh the fixed-stride context and generation page table once per forward.""" - fp4_mla_state = find_fp4_mla(metadata) - if not bool(getattr(fp4_mla_state, "device_page_table", False)): + if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA requires fixed-stride device page metadata.") - if bool(getattr(fp4_mla_state, "device_page_table_valid", False)): + if bool(getattr(metadata.fp4_mla_state, "device_page_table_valid", False)): return num_contexts = int(metadata.num_contexts) num_sequences = int(metadata.num_seqs) num_generation_sequences = num_sequences - num_contexts - max_pages = int(fp4_mla_state.page_table_stride) + max_pages = int(metadata.fp4_mla_state.page_table_stride) if num_sequences <= 0 or max_pages <= 0: raise RuntimeError( "FP4 MLA device page metadata requires positive sequence and page capacities." @@ -667,15 +664,15 @@ def materialize_fp4_mla_device_page_table( f"{num_generation_sequences} entries." ) - cache_pool_id = int(getattr(fp4_mla_state, "cache_pool_id", 0)) + cache_pool_id = int(getattr(metadata.fp4_mla_state, "cache_pool_id", 0)) block_offsets = metadata.kv_cache_block_offsets[ cache_pool_id, :num_sequences, 0, :max_pages, ] - page_ids = fp4_mla_state._paged_kv_indices[: num_sequences * max_pages] - page_index_scale = int(fp4_mla_state.cache_page_index_scale) + page_ids = metadata.fp4_mla_state._paged_kv_indices[: num_sequences * max_pages] + page_index_scale = int(metadata.fp4_mla_state.cache_page_index_scale) if page_index_scale <= 0: raise RuntimeError("FP4 MLA device page metadata requires a positive page-index scale.") grid = ( @@ -684,8 +681,8 @@ def materialize_fp4_mla_device_page_table( ) _fp4_mla_materialize_page_table_kernel[grid]( page_ids, - fp4_mla_state._paged_kv_indptr, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state._paged_kv_indptr, + metadata.fp4_mla_state.paged_kv_indptr_decode, block_offsets, kv_lens, generation_kv_lens, @@ -698,11 +695,11 @@ def materialize_fp4_mla_device_page_table( PAGE_TILE_SIZE=_FP4_MLA_PAGE_TABLE_TILE_SIZE, num_warps=4, ) - hp_page_ids = getattr(fp4_mla_state, "hp_page_indices", None) + hp_page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) if not isinstance(hp_page_ids, torch.Tensor): raise RuntimeError("FP4 MLA requires an HP page-table output tensor.") - hp_pool_id = int(fp4_mla_state.hp_pool_id) - hp_page_index_scale = int(fp4_mla_state.hp_page_index_scale) + hp_pool_id = int(metadata.fp4_mla_state.hp_pool_id) + hp_page_index_scale = int(metadata.fp4_mla_state.hp_page_index_scale) hp_block_offsets = metadata.kv_cache_block_offsets[ hp_pool_id, :num_sequences, @@ -711,8 +708,8 @@ def materialize_fp4_mla_device_page_table( ] _fp4_mla_materialize_page_table_kernel[grid]( hp_page_ids, - fp4_mla_state._paged_kv_indptr, - fp4_mla_state.paged_kv_indptr_decode, + metadata.fp4_mla_state._paged_kv_indptr, + metadata.fp4_mla_state.paged_kv_indptr_decode, hp_block_offsets, kv_lens, generation_kv_lens, @@ -725,7 +722,7 @@ def materialize_fp4_mla_device_page_table( PAGE_TILE_SIZE=_FP4_MLA_PAGE_TABLE_TILE_SIZE, num_warps=4, ) - fp4_mla_state.device_page_table_valid = True + metadata.fp4_mla_state.device_page_table_valid = True def _host_int_list_during_forward(value: Any, start: int, end: int) -> Optional[list[int]]: @@ -742,7 +739,7 @@ def _fp4_mla_generation_num_blocks_device(metadata: Any) -> torch.Tensor: slots are never consumed. """ num_gen = metadata.num_seqs - metadata.num_contexts - return find_fp4_mla(metadata).paged_kv_indptr_decode[num_gen : num_gen + 1] + return metadata.fp4_mla_state.paged_kv_indptr_decode[num_gen : num_gen + 1] def _fp4_mla_uniform_generation_lengths( @@ -757,7 +754,6 @@ def _fp4_mla_uniform_generation_lengths( step's packed split plan (one CuTe DSL kernel instead of two launches), and a stale plan is rewritten even when the lengths are already current. """ - fp4_mla_state = find_fp4_mla(metadata) if num_gen <= 0 or num_gen_tokens % num_gen != 0: raise RuntimeError("FP4 MLA generation requires a non-empty uniform request batch.") if plan_geometry is not None and plan_geometry.num_requests != num_gen: @@ -767,8 +763,8 @@ def _fp4_mla_uniform_generation_lengths( num_seqs = metadata.num_seqs kv_lens_gen = metadata.kv_lens_cuda_runtime[num_contexts:num_seqs] prompt_lens_gen = metadata.prompt_lens_cuda_runtime[num_contexts:num_seqs] - corrected_kv_lens = getattr(fp4_mla_state, "generation_kv_lens", None) - generation_lens = getattr(fp4_mla_state, "generation_append_lens", None) + corrected_kv_lens = getattr(metadata.fp4_mla_state, "generation_kv_lens", None) + generation_lens = getattr(metadata.fp4_mla_state, "generation_append_lens", None) tensors = ( kv_lens_gen, prompt_lens_gen, @@ -786,18 +782,18 @@ def _fp4_mla_uniform_generation_lengths( record_for_capture = bool( getattr(metadata, "is_cuda_graph", False) and torch.cuda.is_current_stream_capturing() - and not getattr(fp4_mla_state, "generation_lengths_capture_recorded", False) + and not getattr(metadata.fp4_mla_state, "generation_lengths_capture_recorded", False) ) precomputed = ( not record_for_capture - and fp4_mla_state.generation_lengths_num_tokens == num_gen_tokens - and fp4_mla_state.generation_lengths_num_seqs == num_gen - and fp4_mla_state.generation_lengths_num_contexts == num_contexts + and metadata.fp4_mla_state.generation_lengths_num_tokens == num_gen_tokens + and metadata.fp4_mla_state.generation_lengths_num_seqs == num_gen + and metadata.fp4_mla_state.generation_lengths_num_contexts == num_contexts ) plan_pending = ( plan_geometry is not None and plan_geometry.key is not None - and fp4_mla_state.packed_plan_key != plan_geometry.key + and metadata.fp4_mla_state.packed_plan_key != plan_geometry.key ) if not precomputed: if plan_pending: @@ -821,11 +817,11 @@ def _fp4_mla_uniform_generation_lengths( num_gen_tokens=num_gen_tokens, num_gen=num_gen, ) - fp4_mla_state.generation_lengths_num_tokens = num_gen_tokens - fp4_mla_state.generation_lengths_num_seqs = num_gen - fp4_mla_state.generation_lengths_num_contexts = num_contexts + metadata.fp4_mla_state.generation_lengths_num_tokens = num_gen_tokens + metadata.fp4_mla_state.generation_lengths_num_seqs = num_gen + metadata.fp4_mla_state.generation_lengths_num_contexts = num_contexts if record_for_capture: - fp4_mla_state.generation_lengths_capture_recorded = True + metadata.fp4_mla_state.generation_lengths_capture_recorded = True elif plan_pending: _launch_fp4_mla_packed_plan(plan_geometry, corrected_kv_lens[:num_gen], metadata) return corrected_kv_lens[:num_gen], generation_lens[:num_gen] @@ -836,10 +832,9 @@ def _materialize_fp4_mla_device_page_table_for_forward( generation_kv_lens: Optional[torch.Tensor] = None, ) -> None: """Materialize all fixed-stride rows from final per-forward device lengths.""" - fp4_mla_state = find_fp4_mla(metadata) - if not bool(getattr(fp4_mla_state, "device_page_table", False)): + if not bool(getattr(metadata.fp4_mla_state, "device_page_table", False)): raise RuntimeError("FP4 MLA cache update requires fixed-stride device page metadata.") - if bool(getattr(fp4_mla_state, "device_page_table_valid", False)): + if bool(getattr(metadata.fp4_mla_state, "device_page_table_valid", False)): return num_contexts = int(metadata.num_contexts) @@ -863,13 +858,12 @@ def _materialize_fp4_mla_device_page_table_for_forward( def _max_generation_pages(metadata: Any) -> int: - fp4_mla_state = find_fp4_mla(metadata) num_gen = metadata.num_seqs - metadata.num_contexts if num_gen <= 0: return 0 - if not getattr(fp4_mla_state, "device_page_table", False): + if not getattr(metadata.fp4_mla_state, "device_page_table", False): raise RuntimeError("FP4 MLA generation requires fixed-stride device page metadata.") - max_pages = int(fp4_mla_state.page_table_stride) + max_pages = int(metadata.fp4_mla_state.page_table_stride) if max_pages <= 0: raise RuntimeError("FP4 MLA device page-table stride must be positive.") return max_pages @@ -884,7 +878,7 @@ def _fp4_mla_generation_page_ids(metadata: Any, num_gen_seqs: int) -> torch.Tens f"{num_gen_seqs} != {expected_num_gen}." ) max_pages = _max_generation_pages(metadata) - page_ids = getattr(find_fp4_mla(metadata), "_paged_kv_indices", None) + page_ids = getattr(metadata.fp4_mla_state, "_paged_kv_indices", None) start = metadata.num_contexts * max_pages end = start + num_gen_seqs * max_pages if ( @@ -907,7 +901,7 @@ def _fp4_mla_generation_hp_page_ids(metadata: Any, num_gen_seqs: int) -> torch.T f"{num_gen_seqs} != {expected_num_gen}." ) max_pages = _max_generation_pages(metadata) - page_ids = getattr(find_fp4_mla(metadata), "hp_page_indices", None) + page_ids = getattr(metadata.fp4_mla_state, "hp_page_indices", None) start = metadata.num_contexts * max_pages end = start + num_gen_seqs * max_pages if ( @@ -935,13 +929,12 @@ def _host_int_list(value: Any, start: int, end: int) -> Optional[list[int]]: def _infer_assume_full_pages(metadata: Any, max_pages: int, page_size: int) -> bool: - fp4_mla_state = find_fp4_mla(metadata) if getattr(metadata, "is_cuda_graph", False): return False start = metadata.num_contexts end = metadata.num_seqs - block_counts = _host_int_list(getattr(fp4_mla_state, "num_blocks", None), start, end) + block_counts = _host_int_list(getattr(metadata.fp4_mla_state, "num_blocks", None), start, end) if block_counts is not None and ( not block_counts or min(block_counts) != max_pages or max(block_counts) != max_pages ): @@ -957,12 +950,12 @@ def _infer_assume_full_pages(metadata: Any, max_pages: int, page_size: int) -> b tuple(block_counts) if block_counts is not None else None, kv_lens_cuda.data_ptr(), ) - cache = getattr(fp4_mla_state, "full_pages_cache", None) + cache = getattr(metadata.fp4_mla_state, "full_pages_cache", None) if cache is not None and cache[0] == cache_key: return bool(cache[1]) kv_lens = [int(item) for item in kv_lens_cuda[start:end].detach().cpu().tolist()] result = bool(kv_lens) and min(kv_lens) == max(kv_lens) == max_pages * page_size - setattr(fp4_mla_state, "full_pages_cache", (cache_key, result)) + setattr(metadata.fp4_mla_state, "full_pages_cache", (cache_key, result)) return result kv_cache_params = getattr(metadata, "kv_cache_params", None) diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py index 411cb9fc300c..8c9817444a50 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/packed_plan.py @@ -18,7 +18,6 @@ _fp4_mla_packed_wave_plan_enabled, ) from .layout import _ensure_workspace_tensor -from .state import find_fp4_mla _SM_COUNT_CACHE: dict[int, int] = {} @@ -105,14 +104,13 @@ def _fp4_mla_packed_plan_geometry( generation lengths (``_fp4_mla_uniform_generation_lengths``). ``key`` is None when every request keeps a single task and no plan is needed. """ - fp4_mla_state = find_fp4_mla(metadata) rows_per_seq = _fp4_mla_packed_rows_per_seq(num_heads, query_len) if rows_per_seq == 0 or num_requests <= 0: return None - if not fp4_mla_state.device_page_table: + if not metadata.fp4_mla_state.device_page_table: return None # The fixed-stride generation page-table capacity (see _max_generation_pages). - max_pages = int(fp4_mla_state.page_table_stride) + max_pages = int(metadata.fp4_mla_state.page_table_stride) if max_pages <= 0: return None max_kv_len = max_pages * metadata.page_size @@ -166,7 +164,7 @@ def _fp4_mla_packed_plan_geometry( dtype=torch.int32, device=device, ) - kv_lens = fp4_mla_state.generation_kv_lens + kv_lens = metadata.fp4_mla_state.generation_kv_lens key = ( num_requests, splits, @@ -232,4 +230,4 @@ def _launch_fp4_mla_packed_plan( generation_lengths=generation_lengths, wave_plan=_fp4_mla_packed_wave_plan_enabled(), ) - find_fp4_mla(metadata).packed_plan_key = geometry.key + metadata.fp4_mla_state.packed_plan_key = geometry.key diff --git a/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py b/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py index 32035bfd5230..5c92b130ea0b 100644 --- a/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py +++ b/tensorrt_llm/_torch/attention/backends/fp4_mla/v_cache.py @@ -26,7 +26,6 @@ ) from .fp4_mla_kernels import _fp4_mla_rebuild_v_scale_from_k_scale_kernel from .layout import _ensure_workspace_tensor, get_fp4_mla_v_scale_pool_size -from .state import find_fp4_mla def _shared_v_pack_storage_enabled() -> bool: @@ -256,7 +255,7 @@ def _set_triton_v_packed_cache_valid( if _shared_v_pack_storage_enabled() else _triton_v_packed_valid_attr(layer_idx) ) - find_fp4_mla(metadata).v_packed_cache_tags[valid_attr] = _triton_v_packed_cache_tag( + metadata.fp4_mla_state.v_packed_cache_tags[valid_attr] = _triton_v_packed_cache_tag( layer_idx, kv_cache, v_head_dim=v_head_dim, @@ -285,7 +284,7 @@ def _is_triton_v_packed_cache_valid( if _shared_v_pack_storage_enabled() else _triton_v_packed_valid_attr(layer_idx) ) - return find_fp4_mla(metadata).v_packed_cache_tags.get(valid_attr) == _triton_v_packed_cache_tag( + return metadata.fp4_mla_state.v_packed_cache_tags.get(valid_attr) == _triton_v_packed_cache_tag( layer_idx, kv_cache, v_head_dim=v_head_dim, @@ -323,7 +322,7 @@ def _get_triton_v_packed_cache( page_ids=page_ids, ): return None - v_packed = find_fp4_mla(metadata).workspaces.get(_triton_v_packed_attr(layer_idx)) + v_packed = metadata.fp4_mla_state.workspaces.get(_triton_v_packed_attr(layer_idx)) expected_shape = _v_packed_shape(kv_cache, v_head_dim, page_size, block_v) if ( v_packed is None diff --git a/tensorrt_llm/_torch/attention/backends/interface.py b/tensorrt_llm/_torch/attention/backends/interface.py index 38c08f1dfc72..61860e0ab481 100644 --- a/tensorrt_llm/_torch/attention/backends/interface.py +++ b/tensorrt_llm/_torch/attention/backends/interface.py @@ -232,6 +232,140 @@ class AttentionMetadata: # The attached extensions, the core first; see extensions.py. _extensions: ExtensionRegistry + def get_adp_token_counts(self) -> List[int]: + """Local execution-stage counts carried by the input count exchange. + + The first stage is the input (``num_tokens``, unless an extension + declares ``adp_input_token_count``); each extension may append its own. + """ + input_tokens = self.find("adp_input_token_count", None) + if input_tokens is None: + input_tokens = self.num_tokens + return [input_tokens, *(self.find("adp_token_counts", None) or ())] + + def set_adp_token_counts(self, counts: List[List[int]]) -> List[int]: + """Save stage counts and return the input-stage counts for padding.""" + if len(counts) > 1: + self.find("receive_adp_token_counts")(counts[1:], counts[0]) + return counts[0] + + def find(self, name: str, default: Any = None) -> Any: + """An attached extension's declared field or bound method, or ``default``. + + For values the core or the engine reads from whichever extension + provides them, such as ``kv_cache_pools`` or ``full_block_table_width``. + """ + for extension in self._attached_extensions(): + declared = type(extension) + if name in declared.declared_fields( + ) or name in declared.declared_methods(): + return getattr(extension, name) + return default + + @classmethod + def core_class(cls) -> type: + """The metadata family class, without the forwarding of a composed class.""" + return cls.__dict__.get("_core_class", cls) + + def _compose(self) -> None: + """Switch to the composed class of the attached extensions. + + The composed class forwards each extension's declared fields and + methods, so callers read ``metadata.field`` instead of + ``metadata.ext(T).field``. + """ + extension_types = tuple(self._attached_extensions().types()) + object.__setattr__(self, "__class__", + _composed_class(self.core_class(), extension_types)) + + def own_attribute_names(self) -> FrozenSet[str]: + """Names this metadata's family defines; extensions may not shadow them.""" + return _assignable_names(self.core_class()) + + def _attached_extensions(self) -> ExtensionRegistry: + # Metadata built without __init__ (as some tests do) has no registry. + return self.__dict__.get("_extensions", _NO_EXTENSIONS) + + def ext(self, cls: Type[ExtensionT]) -> ExtensionT: + """The attached extension of type ``cls``; raises when none is.""" + found = self._attached_extensions().find(cls) + if found is None: + raise LookupError( + f"{type(self).__name__} has no {cls.__name__} extension attached." + ) + return found + + def find_ext(self, cls: Type[ExtensionT]) -> Optional[ExtensionT]: + """The attached extension of type ``cls``, or None.""" + return self._attached_extensions().find(cls) + + def add_ext(self, cls: Type[ExtensionT]) -> ExtensionT: + """Construct and attach ``cls`` and its requirements if absent; return the extension.""" + self._ensure_core() + self._extensions.attach(self, (cls, )) + return self.ext(cls) + + def replace_ext(self, extension: MetadataExtension) -> None: + """Attach a built extension in place of any of its type. + + For derived views made by ``copy.copy``: binds this metadata to its own + registry first, so the source keeps its extensions. + """ + assert extension.metadata is self, "the extension must be built for this metadata" + if not extension.is_core: + self._ensure_core() + elif self.__dict__.get( + "_extensions") is None or self._extensions.owner is not self: + self._extensions = self._attached_extensions().bound_to(self) + self._extensions.replace(extension) + self._compose() + + def remove_ext(self, cls: Type[MetadataExtension]) -> None: + """Detach the extension of type ``cls`` from a derived view. + + Like ``replace_ext``, binds this metadata to its own registry first. + """ + self._ensure_core() + self._extensions.remove(cls) + self._compose() + + @property + def extensions(self) -> List[MetadataExtension]: + """The attached extensions in dependency order.""" + return list(self._attached_extensions()) + + def get_buffer(self, shape: Sequence[int], dtype: torch.dtype, + name: str) -> torch.Tensor: + """A persistent buffer from this metadata's pool, graph-stable under capture.""" + return self.get_empty(self.cuda_graph_buffers, + list(shape), + dtype=dtype, + cache_name=name, + capture_graph=self.is_cuda_graph) + + def stage_to_device(self, key: Any, destination: torch.Tensor, + value: torch.Tensor) -> None: + """Copy a host tensor into ``destination`` through pinned staging.""" + copy_host_to_device(self.core()._host_staging, key, destination, value) + + def adopts_target_state(self) -> bool: + """Whether every attached extension takes a draft's state from its target's.""" + return all( + type(extension).adopts_target_state + for extension in self._attached_extensions()) + + def check_lengths_current(self) -> None: + """Raise when the lengths changed in place without a refresh. + + The attention forward calls it: state derived from the lengths would + be stale. + """ + if self.lengths_changed: + raise RuntimeError( + f"{type(self).__name__}'s lengths changed in place since its " + "last prepare or draft refresh; refresh the draft before its " + "forward") + def __init__(self, *, extension_types: Sequence[Type[MetadataExtension]] = (), @@ -403,140 +537,6 @@ def _ensure_core(self) -> AttentionCoreMetadata: None) return core - def get_adp_token_counts(self) -> List[int]: - """Local execution-stage counts carried by the input count exchange. - - The first stage is the input (``num_tokens``, unless an extension - declares ``adp_input_token_count``); each extension may append its own. - """ - input_tokens = self.find("adp_input_token_count", None) - if input_tokens is None: - input_tokens = self.num_tokens - return [input_tokens, *(self.find("adp_token_counts", None) or ())] - - def set_adp_token_counts(self, counts: List[List[int]]) -> List[int]: - """Save stage counts and return the input-stage counts for padding.""" - if len(counts) > 1: - self.find("receive_adp_token_counts")(counts[1:], counts[0]) - return counts[0] - - def find(self, name: str, default: Any = None) -> Any: - """An attached extension's declared field or bound method, or ``default``. - - For values the core or the engine reads from whichever extension - provides them, such as ``kv_cache_pools`` or ``full_block_table_width``. - """ - for extension in self._attached_extensions(): - declared = type(extension) - if name in declared.declared_fields( - ) or name in declared.declared_methods(): - return getattr(extension, name) - return default - - @classmethod - def core_class(cls) -> type: - """The metadata family class, without the forwarding of a composed class.""" - return cls.__dict__.get("_core_class", cls) - - def _compose(self) -> None: - """Switch to the composed class of the attached extensions. - - The composed class forwards each extension's declared fields and - methods, so callers read ``metadata.field`` instead of - ``metadata.ext(T).field``. - """ - extension_types = tuple(self._attached_extensions().types()) - object.__setattr__(self, "__class__", - _composed_class(self.core_class(), extension_types)) - - def own_attribute_names(self) -> FrozenSet[str]: - """Names this metadata's family defines; extensions may not shadow them.""" - return _assignable_names(self.core_class()) - - def _attached_extensions(self) -> ExtensionRegistry: - # Metadata built without __init__ (as some tests do) has no registry. - return self.__dict__.get("_extensions", _NO_EXTENSIONS) - - def ext(self, cls: Type[ExtensionT]) -> ExtensionT: - """The attached extension of type ``cls``; raises when none is.""" - found = self._attached_extensions().find(cls) - if found is None: - raise LookupError( - f"{type(self).__name__} has no {cls.__name__} extension attached." - ) - return found - - def find_ext(self, cls: Type[ExtensionT]) -> Optional[ExtensionT]: - """The attached extension of type ``cls``, or None.""" - return self._attached_extensions().find(cls) - - def add_ext(self, cls: Type[ExtensionT]) -> ExtensionT: - """Construct and attach ``cls`` and its requirements if absent; return the extension.""" - self._ensure_core() - self._extensions.attach(self, (cls, )) - return self.ext(cls) - - def replace_ext(self, extension: MetadataExtension) -> None: - """Attach a built extension in place of any of its type. - - For derived views made by ``copy.copy``: binds this metadata to its own - registry first, so the source keeps its extensions. - """ - assert extension.metadata is self, "the extension must be built for this metadata" - if not extension.is_core: - self._ensure_core() - elif self.__dict__.get( - "_extensions") is None or self._extensions.owner is not self: - self._extensions = self._attached_extensions().bound_to(self) - self._extensions.replace(extension) - self._compose() - - def remove_ext(self, cls: Type[MetadataExtension]) -> None: - """Detach the extension of type ``cls`` from a derived view. - - Like ``replace_ext``, binds this metadata to its own registry first. - """ - self._ensure_core() - self._extensions.remove(cls) - self._compose() - - @property - def extensions(self) -> List[MetadataExtension]: - """The attached extensions in dependency order.""" - return list(self._attached_extensions()) - - def get_buffer(self, shape: Sequence[int], dtype: torch.dtype, - name: str) -> torch.Tensor: - """A persistent buffer from this metadata's pool, graph-stable under capture.""" - return self.get_empty(self.cuda_graph_buffers, - list(shape), - dtype=dtype, - cache_name=name, - capture_graph=self.is_cuda_graph) - - def stage_to_device(self, key: Any, destination: torch.Tensor, - value: torch.Tensor) -> None: - """Copy a host tensor into ``destination`` through pinned staging.""" - copy_host_to_device(self.core()._host_staging, key, destination, value) - - def adopts_target_state(self) -> bool: - """Whether every attached extension takes a draft's state from its target's.""" - return all( - type(extension).adopts_target_state - for extension in self._attached_extensions()) - - def check_lengths_current(self) -> None: - """Raise when the lengths changed in place without a refresh. - - The attention forward calls it: state derived from the lengths would - be stale. - """ - if self.lengths_changed: - raise RuntimeError( - f"{type(self).__name__}'s lengths changed in place since its " - "last prepare or draft refresh; refresh the draft before its " - "forward") - @property def has_cross_sub_metadata(self) -> bool: """Whether this metadata carries a cross-attention sub-metadata.""" diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py index 6580bb9bf79d..f393ae334e44 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py @@ -82,17 +82,7 @@ if TYPE_CHECKING: from tensorrt_llm.mapping import Mapping - from .metadata import DSAMetadata, DSAtrtllmAttentionMetadata - - -def _dsa_state(metadata: "DSAtrtllmAttentionMetadata") -> "DSAMetadata": - """The DSA extension of ``metadata``. - - Imported on use: the metadata module imports this one. - """ - from .metadata import DSAMetadata - - return metadata.ext(DSAMetadata) + from .metadata import DSAtrtllmAttentionMetadata # Optional import: fast-hadamard-transform causes CI build issues (requires wheel+torch pre-installed) @@ -320,32 +310,31 @@ def transform_local_topk_and_prepare_pool_view( Uses cached values from attn_metadata._ensure_pool_view_cached() to avoid redundant Python/CUDA overhead across layers. """ - dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - dsa._ensure_pool_view_cached() + attn_metadata._ensure_pool_view_cached() page_index_scale, layer_offset = ( attn_metadata.kv_cache_manager.get_primary_pool_page_index_params(layer_idx) ) if is_generation: - block_table = dsa._cached_block_table_gen - req_idx = dsa._cached_req_idx_gen + block_table = attn_metadata._cached_block_table_gen + req_idx = attn_metadata._cached_req_idx_gen else: - block_table = dsa._cached_block_table_ctx - req_idx = dsa._cached_req_idx_ctx + block_table = attn_metadata._cached_block_table_ctx + req_idx = attn_metadata._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global( req_idx, block_table, topk_indices, - dsa._cached_tokens_per_block, + attn_metadata._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * dsa._cached_tokens_per_block, + page_index_scale * attn_metadata._cached_tokens_per_block, layer_offset, ) - return global_indices, dsa._cached_pool_view + return global_indices, attn_metadata._cached_pool_view def transform_local_topk_and_prepare_pool_view_grouped( @@ -370,25 +359,24 @@ def transform_local_topk_and_prepare_pool_view_grouped( is bit-identical to ``transform_local_topk_and_prepare_pool_view`` for the member with offset ``layer_ids[z]``. MLA-only path. """ - dsa = _dsa_state(attn_metadata) assert topk_indices.dtype == torch.int32 - dsa._ensure_pool_view_cached() + attn_metadata._ensure_pool_view_cached() if is_generation: - block_table = dsa._cached_block_table_gen - req_idx = dsa._cached_req_idx_gen + block_table = attn_metadata._cached_block_table_gen + req_idx = attn_metadata._cached_req_idx_gen else: - block_table = dsa._cached_block_table_ctx - req_idx = dsa._cached_req_idx_ctx + block_table = attn_metadata._cached_block_table_ctx + req_idx = attn_metadata._cached_req_idx_ctx global_indices = torch.ops.trtllm.convert_req_index_to_global_grouped( req_idx, block_table, topk_indices, - dsa._cached_tokens_per_block, + attn_metadata._cached_tokens_per_block, topk_indices.shape[1], - page_index_scale * dsa._cached_tokens_per_block, + page_index_scale * attn_metadata._cached_tokens_per_block, layer_ids, ) @@ -1183,7 +1171,6 @@ def prepare_one_prefill_chunk( @staticmethod def build_indexer_params(metadata: DSAtrtllmAttentionMetadata) -> Optional[IndexerParams]: - dsa = _dsa_state(metadata) kv_cache_manager = metadata.kv_cache_manager if kv_cache_manager is None or not hasattr(kv_cache_manager, "index_head_dim"): return None @@ -1201,7 +1188,7 @@ def build_indexer_params(metadata: DSAtrtllmAttentionMetadata) -> Optional[Index num_ctx_tokens=metadata.num_ctx_tokens, head_dim=head_dim, quant_block_size=metadata.indexer_quant_block_size, - tokens_per_block=dsa._tokens_per_block, + tokens_per_block=metadata._tokens_per_block, compress_ratio=compress_ratio, request_ids=metadata.request_ids, num_past_tokens=metadata.kv_cache_params.num_cached_tokens_per_seq, @@ -1444,10 +1431,9 @@ def prepare_scheduler_metadata(metadata: DSAtrtllmAttentionMetadata): """ Prepare scheduler metadata for the DeepGEMM decode MQA kernel. """ - dsa = _dsa_state(metadata) num_contexts = metadata.num_contexts num_generations = metadata.num_generations - gen_seq_lens = dsa.bind_gen_indexer_kv_lens( + gen_seq_lens = metadata.bind_gen_indexer_kv_lens( metadata.kv_lens_cuda_runtime[num_contexts : num_contexts + num_generations] ) if not metadata.use_expanded_buffers_for_mtp: diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py index fdde42abb484..129dfd8929b3 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/metadata.py @@ -13,7 +13,6 @@ import tensorrt_llm import tensorrt_llm.bindings -from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from tensorrt_llm._torch.utils import maybe_compile @@ -224,6 +223,7 @@ class DSAMetadata(SparseAlgorithmExtension, MTPIndexShareUser): ) methods = frozenset( { + "bind_gen_indexer_kv_lens", "_compute_kv_lens_row_reorder", "_create_kv_lens_2d_buffer", "_ensure_group_remap_struct", @@ -406,6 +406,111 @@ class DSAMetadata(SparseAlgorithmExtension, MTPIndexShareUser): # overlays the high pool's rows on it. mixed_precision_staging_enabled = False + @property + def is_ragged_verify(self) -> bool: + return self.ragged_verify_lens is not None + + def set_gen_token_layout( + self, + *, + tokens_per_gen_step: int, + ragged_verify_lens: Optional[List[int]], + device_windows_mode: bool, + ) -> None: + """Take this step's generation-token layout; read by the next ``prepare``. + + ``tokens_per_gen_step`` is the uniform query width of a generation + request at the step's runtime draft length (0: the static draft-token + ceiling); ``ragged_verify_lens`` the per-request widths of a ragged + verify step, or None; ``device_windows_mode`` whether those widths are + only the captured shape split, the true windows arriving on device + through ``apply_device_ragged_layout`` after prepare. + """ + self.runtime_tokens_per_gen_step = tokens_per_gen_step + self.ragged_verify_lens = ragged_verify_lens + self.device_windows_mode = device_windows_mode + + def set_ragged_verify_lens(self, ragged_verify_lens: Optional[List[int]]) -> None: + """Take this step's per-request verify widths (None: a uniform step).""" + self.ragged_verify_lens = ragged_verify_lens + + def _validate_ragged_verification(self) -> None: + """Reject layouts whose stable buffers were not enabled at allocation.""" + if self.ragged_verify_lens is not None and not self.enable_ragged_verification: + raise RuntimeError( + "ragged verification metadata was installed without enabling " + "ragged-verification buffers" + ) + + @property + def gen_token_stride(self) -> int: + stride = int(self.runtime_tokens_per_gen_step) + return stride if stride > 0 else 1 + self.max_draft_tokens + + def update_indexer_kv_lens_2d(self, gen_kv_lens: torch.Tensor) -> torch.Tensor: + """Fill and return the active query-width view of persistent storage.""" + md = self.metadata + width = min(self.gen_token_stride, self.kv_lens_cuda_2d.shape[1]) + context_lens = self.kv_lens_cuda_2d[: md.num_generations, :width] + context_lens.copy_(gen_kv_lens.unsqueeze(-1).expand(-1, width)) + return context_lens + + def mla_scheduler_buffers(self, num_heads: int) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """Precompute prefixes for the same rows presented to MLA generation.""" + view = self.token_major_gen_view() + if view is None: + return None + + num_ctx = self.metadata.num_contexts + n_gen = view.num_rows - num_ctx + if not self._ragged_mla_buffers_valid: + # RoPE presents one query token per generation row. The device + # extents are causal, pre-extra-KV lengths, just like the + # cache_seq_lens scanned by the non-precomputed kernel path. + torch.cumsum( + view.prompt_lens_cuda[num_ctx:], + 0, + dtype=torch.int32, + out=self.ragged_mla_cu_q_rows[1 : n_gen + 1], + ) + self.ragged_mla_cu_q_rows[1 : n_gen + 1] *= num_heads + torch.cumsum( + view.sequence_length[num_ctx:], + 0, + dtype=torch.int32, + out=self.ragged_mla_cu_kv_seqlens[1 : n_gen + 1], + ) + self._ragged_mla_buffers_valid = True + return ( + self.ragged_mla_cu_q_rows[: n_gen + 1], + self.ragged_mla_cu_kv_seqlens[: n_gen + 1], + ) + + def gen_token_repeat_list(self) -> List[int]: + md = self.metadata + if not self.is_ragged_verify: + return [self.gen_token_stride] * md.num_generations + return list(self.ragged_verify_lens) + + def gen_token_repeats(self, device: Optional[torch.device] = None) -> torch.Tensor: + return torch.tensor(self.gen_token_repeat_list(), dtype=torch.long, device=device) + + def expand_per_gen_token(self, values: torch.Tensor, dim: int = 0) -> Tuple[torch.Tensor, int]: + """Repeat request values once per generation query token.""" + md = self.metadata + repeats = self.gen_token_repeat_list() + num_tokens = sum(repeats) + if not self.is_ragged_verify: + return values.repeat_interleave(self.gen_token_stride, dim=dim), num_tokens + if values.device == self.gen_token_repeats_cuda.device: + repeats_dev = self.gen_token_repeats_cuda[: md.num_generations] + else: + repeats_dev = torch.tensor(repeats, dtype=torch.long, device=values.device) + return ( + values.repeat_interleave(repeats_dev, dim=dim, output_size=num_tokens), + num_tokens, + ) + def __init__(self, metadata, *, source=None, **config) -> None: super().__init__(metadata, source=source, **config) self.use_fp8_ds_mla = False @@ -588,6 +693,14 @@ def __init__(self, metadata, *, source=None, **config) -> None: self._create_nvfp4_mla_generation_buffers(capture_graph=capture_graph) self._init_dsa_sparse_offload_state() + @property + def use_cute_dsl_paged_mqa_logits(self) -> bool: + """The Indexer's decision: the config flag gated on SM100 and cutlass-dsl.""" + md = self.metadata + return use_cute_dsl_paged_mqa_logits( + md.sparse_metadata_params.use_cute_dsl_paged_mqa_logits + ) + def _offloads_dsa_latent(self) -> bool: """Whether attention reads a DSA cache manager's MLA latent through the offload arena.""" md = self.metadata @@ -711,119 +824,6 @@ def _fill_mixed_precision_high_block_tables(self) -> None: high_layer = next(iter(manager.mixed_precision_high_layers.values())) self._fill_mixed_precision_high_block_table(manager, high_layer, high_table) - @property - def is_ragged_verify(self) -> bool: - return self.ragged_verify_lens is not None - - def set_gen_token_layout( - self, - *, - tokens_per_gen_step: int, - ragged_verify_lens: Optional[List[int]], - device_windows_mode: bool, - ) -> None: - """Take this step's generation-token layout; read by the next ``prepare``. - - ``tokens_per_gen_step`` is the uniform query width of a generation - request at the step's runtime draft length (0: the static draft-token - ceiling); ``ragged_verify_lens`` the per-request widths of a ragged - verify step, or None; ``device_windows_mode`` whether those widths are - only the captured shape split, the true windows arriving on device - through ``apply_device_ragged_layout`` after prepare. - """ - self.runtime_tokens_per_gen_step = tokens_per_gen_step - self.ragged_verify_lens = ragged_verify_lens - self.device_windows_mode = device_windows_mode - - def set_ragged_verify_lens(self, ragged_verify_lens: Optional[List[int]]) -> None: - """Take this step's per-request verify widths (None: a uniform step).""" - self.ragged_verify_lens = ragged_verify_lens - - def _validate_ragged_verification(self) -> None: - """Reject layouts whose stable buffers were not enabled at allocation.""" - if self.ragged_verify_lens is not None and not self.enable_ragged_verification: - raise RuntimeError( - "ragged verification metadata was installed without enabling " - "ragged-verification buffers" - ) - - @property - def gen_token_stride(self) -> int: - stride = int(self.runtime_tokens_per_gen_step) - return stride if stride > 0 else 1 + self.max_draft_tokens - - def update_indexer_kv_lens_2d(self, gen_kv_lens: torch.Tensor) -> torch.Tensor: - """Fill and return the active query-width view of persistent storage.""" - md = self.metadata - width = min(self.gen_token_stride, self.kv_lens_cuda_2d.shape[1]) - context_lens = self.kv_lens_cuda_2d[: md.num_generations, :width] - context_lens.copy_(gen_kv_lens.unsqueeze(-1).expand(-1, width)) - return context_lens - - def mla_scheduler_buffers(self, num_heads: int) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """Precompute prefixes for the same rows presented to MLA generation.""" - view = self.token_major_gen_view() - if view is None: - return None - - num_ctx = self.metadata.num_contexts - n_gen = view.num_rows - num_ctx - if not self._ragged_mla_buffers_valid: - # RoPE presents one query token per generation row. The device - # extents are causal, pre-extra-KV lengths, just like the - # cache_seq_lens scanned by the non-precomputed kernel path. - torch.cumsum( - view.prompt_lens_cuda[num_ctx:], - 0, - dtype=torch.int32, - out=self.ragged_mla_cu_q_rows[1 : n_gen + 1], - ) - self.ragged_mla_cu_q_rows[1 : n_gen + 1] *= num_heads - torch.cumsum( - view.sequence_length[num_ctx:], - 0, - dtype=torch.int32, - out=self.ragged_mla_cu_kv_seqlens[1 : n_gen + 1], - ) - self._ragged_mla_buffers_valid = True - return ( - self.ragged_mla_cu_q_rows[: n_gen + 1], - self.ragged_mla_cu_kv_seqlens[: n_gen + 1], - ) - - def gen_token_repeat_list(self) -> List[int]: - md = self.metadata - if not self.is_ragged_verify: - return [self.gen_token_stride] * md.num_generations - return list(self.ragged_verify_lens) - - def gen_token_repeats(self, device: Optional[torch.device] = None) -> torch.Tensor: - return torch.tensor(self.gen_token_repeat_list(), dtype=torch.long, device=device) - - def expand_per_gen_token(self, values: torch.Tensor, dim: int = 0) -> Tuple[torch.Tensor, int]: - """Repeat request values once per generation query token.""" - md = self.metadata - repeats = self.gen_token_repeat_list() - num_tokens = sum(repeats) - if not self.is_ragged_verify: - return values.repeat_interleave(self.gen_token_stride, dim=dim), num_tokens - if values.device == self.gen_token_repeats_cuda.device: - repeats_dev = self.gen_token_repeats_cuda[: md.num_generations] - else: - repeats_dev = torch.tensor(repeats, dtype=torch.long, device=values.device) - return ( - values.repeat_interleave(repeats_dev, dim=dim, output_size=num_tokens), - num_tokens, - ) - - @property - def use_cute_dsl_paged_mqa_logits(self) -> bool: - """The Indexer's decision: the config flag gated on SM100 and cutlass-dsl.""" - md = self.metadata - return use_cute_dsl_paged_mqa_logits( - md.sparse_metadata_params.use_cute_dsl_paged_mqa_logits - ) - def prepare(self) -> None: md = self.metadata self._invalidate_length_prefixes() @@ -843,12 +843,11 @@ def prepare(self) -> None: dtype=torch.int, device="cpu", ) - helix = md.find_ext(HelixMetadata) - if helix is not None: + if md.enable_helix: # For Helix CP, inactive ranks only attend to previously cached # tokens (no new token appended), while active ranks add new tokens. # This mirrors the kv_lens logic in TrtllmAttentionMetadata.prepare(). - active_rank = ~helix.helix_is_inactive_rank_cpu[: md.num_seqs] + active_rank = ~md.helix_is_inactive_rank_cpu[: md.num_seqs] kv_lens = cached_token_lens.clone() kv_lens[active_rank] += md.seq_lens_kv[active_rank] else: @@ -880,7 +879,7 @@ def prepare(self) -> None: self.sparse_offload_state, md.request_ids, beam_width=md.beam_width ) - if md.kv_lens_offsets is not None or (helix is not None and md._helix_spec_tokens_valid): + if md.kv_lens_offsets is not None or (md.enable_helix and md._helix_spec_tokens_valid): # The core's device KV lengths carry corrections the host values # above do not (the overlap scheduler's, or Helix's rank-local # lengths of speculative verify groups, prepared before this @@ -2444,9 +2443,8 @@ def prepare_for_indexer_k_cache(self): dtype=torch.int, device="cpu", ) - helix = md.find_ext(HelixMetadata) - if helix is not None: - active_rank = ~helix.helix_is_inactive_rank_cpu[: md.num_seqs] + if md.enable_helix: + active_rank = ~md.helix_is_inactive_rank_cpu[: md.num_seqs] kv_lens = cached_token_lens.clone() kv_lens[active_rank] += md.seq_lens_kv[active_rank] else: diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py index 083a34762c2e..17023d7f4061 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_backend.py @@ -233,6 +233,7 @@ class MiniMaxM3MsaMetadata(SparseAlgorithmExtension): ) methods = frozenset( { + "on_update_kv_lens", "_build_msa_fields", "_build_step_plans", "_create_msa_buffers", diff --git a/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py index e4fb4ccf680e..ca1682fb4b74 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/rocket/metadata.py @@ -72,12 +72,6 @@ class RocketMetadata(SparseAlgorithmExtension): def _predicts_on_device(self) -> bool: return isinstance(self.metadata, TrtllmAttentionMetadata) - @property - def kt_tokens_per_block(self) -> Optional[int]: - """The number of KT tokens per block from the KV cache manager.""" - kv_cache_manager = self.metadata.kv_cache_manager - return kv_cache_manager.kt_tokens_per_block if kv_cache_manager is not None else None - def __init__(self, metadata, *, source=None, **config) -> None: super().__init__(metadata, source=source, **config) buffers = metadata.cuda_graph_buffers @@ -189,6 +183,12 @@ def _budgeted_cached_token_lens(self, batch_params: KVCacheParams) -> list[int]: cached[i] += self.prompt_budget - md.prompt_lens[i] return cached + @property + def kt_tokens_per_block(self) -> Optional[int]: + """The number of KT tokens per block from the KV cache manager.""" + kv_cache_manager = self.metadata.kv_cache_manager + return kv_cache_manager.kt_tokens_per_block if kv_cache_manager is not None else None + def prepare(self) -> None: md = self.metadata if not (self._predicts_on_device and md.kv_cache_manager is None): diff --git a/tensorrt_llm/_torch/attention/backends/trtllm.py b/tensorrt_llm/_torch/attention/backends/trtllm.py index 289d65195b2e..5f21b9a08e4d 100644 --- a/tensorrt_llm/_torch/attention/backends/trtllm.py +++ b/tensorrt_llm/_torch/attention/backends/trtllm.py @@ -367,6 +367,149 @@ class TrtllmCoreMetadata(AttentionCoreMetadata): "num_sparse_topk": 0, } + @property + def effective_beam_width(self) -> int: + # Only use this for the fallback kernel's beam_width argument. + # Cross-attention reads request-scoped encoder K/V that is written once + # and reused unchanged by every decoder beam. Metadata preparation still + # uses beam_width to expand cross block-offset rows to decoder-sequence + # scope, but the fallback kernel should treat the cross K/V cache as + # non-beam-packed. + return 1 if self.is_cross else self.beam_width + + @property + def effective_workspace(self) -> Optional[torch.Tensor]: + """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" + return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace + + @property + def max_context_length(self) -> int: + """ + Upper bound for a single context window. + Required max_seq_len for context-only attention cases like visual gen + """ + return min(self.max_seq_len, self.max_num_tokens) + + @property + def max_seq_len(self) -> int: + """ + Returns the max sequence length. + If the attention uses KV cache, it will return max_seq_len from the KV cache manager. + If the attention is no cache, max_seq_len should be set manually by user. + """ + if self.kv_cache_manager is not None: + return self.kv_cache_manager.max_seq_len + else: + assert self._max_seq_len_storage is not None, "max_seq_len should be set for no kv cache attention" + return self._max_seq_len_storage + + @max_seq_len.setter + def max_seq_len(self, value: int) -> None: + """ + Set the max sequence length for no cache attention. + """ + self._max_seq_len_storage = value + + # Block tables given to a metadata without a KV cache manager; with one, + # the core allocates or derives them from the manager. + _manager_block_tables = frozenset({ + "kv_cache_block_offsets", + "host_kv_cache_block_offsets", + "block_ids_per_seq", + "kv_block_ids_per_seq", + }) + + def creation_value(self, name: str) -> Any: + # The value set for no-cache attention; a cache's comes from its manager. + if name == "max_seq_len": + return self._max_seq_len_storage + if name in self._manager_block_tables and self.kv_cache_manager is not None: + return None + return super().creation_value(name) + + @property + def tokens_per_block(self) -> Optional[int]: + """ + Returns the number of tokens per block from the KV cache manager. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[2] + return self.kv_cache_manager.tokens_per_block if self.kv_cache_manager is not None else None + + @property + def page_size(self) -> int: + """ + Number of tokens per cache page. + """ + page_size_override = getattr(self, '_page_size_override', None) + if page_size_override is not None: + return page_size_override + assert self.kv_cache_manager is not None, "page_size requires a KV cache manager" + return self.kv_cache_manager.tokens_per_block + + @page_size.setter + def page_size(self, value: int) -> None: + self._page_size_override = value + + @property + def host_kv_cache_pool_pointers(self) -> Optional[torch.Tensor]: + """ + Returns the host KV cache pool pointers from the KV cache manager if KV cache manager is not None. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[0] + return self.kv_cache_manager.kv_cache_pool_pointers if self.kv_cache_manager is not None else None + + @property + def host_kv_cache_pool_mapping(self) -> Optional[torch.Tensor]: + """ + Returns the host KV cache pool mapping from the KV cache manager if KV cache manager is not None. + """ + pools = self._extension_kv_cache_pools() + if pools is not None: + return pools[1] + return self.kv_cache_manager.kv_cache_pool_mapping if self.kv_cache_manager is not None else None + + def _extension_kv_cache_pools( + self) -> Optional[Tuple[torch.Tensor, torch.Tensor, int]]: + # An extension's pools (CSA2 compute tiles) replace the manager's. + return self.metadata.find("kv_cache_pools", None) + + def copy_lengths_from(self, source: "AttentionCoreMetadata") -> None: + super().copy_lengths_from(source) + num_seqs = self.num_seqs + self.kv_lens_cuda[:num_seqs].copy_(source.kv_lens_cuda[:num_seqs]) + if self.host_request_types is not None: + self.host_request_types[:num_seqs].copy_( + source.host_request_types[:num_seqs]) + + def resync_length_derived_state(self) -> None: + # A draft step changed the lengths the layer-invariant prefixes derive + # from; the next MLA layer rebuilds them, and the next context-only + # call plans again. + self.context_plan = None + self._invalidate_mla_scheduler_buffers() + self._sync_mixed_precision_staging_lengths() + + def _sync_mixed_precision_staging_lengths(self) -> None: + """Keep the mixed-precision staging lengths in step with kv_lens_cuda. + + The staging copy loads the host lengths in ``prepare_mixed_precision_kv``; + the step's device corrections (``kv_lens_offsets``) and a draft loop's + length changes reach only kv_lens_cuda, so the staging lengths are + taken from it on the device. + """ + view = self.mixed_precision_staging + if (view is None or view.lengths is None + or view.manager is not self.kv_cache_manager): + return + lengths = view.lengths + n = lengths.shape[1] + lengths[1].copy_(self.kv_lens_cuda[:n]) + torch.sub(lengths[1], self.seq_lens_kv_cuda[:n], out=lengths[0]) + def __init__(self, metadata, *, source=None, **config) -> None: super().__init__(metadata, source=source, **config) self.workspace_reclaimable = True @@ -492,371 +635,93 @@ def _allocate_buffers(self, buffers) -> None: dtype=torch.int8, ) - if self.cuda_graph_workspace is None: - self.cuda_graph_workspace = torch.empty( - (0, ), - device='cuda', - dtype=torch.int8, - ) - - if self.kv_cache_manager is not None: - num_attention_op_pools = getattr(self.kv_cache_manager, - "num_attention_op_pools", - self.kv_cache_manager.num_pools) - self.kv_cache_block_offsets = md.get_empty( - buffers, - [ - num_attention_op_pools, self.max_num_sequences, 2, - self.kv_cache_manager.max_blocks_per_seq - ], - cache_name="kv_cache_block_offsets", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.host_kv_cache_block_offsets = self.kv_cache_manager.host_kv_cache_block_offsets - self.block_ids_per_seq = None - self.kv_block_ids_per_seq = None - - if self.enable_context_mla_with_cached_kv: - # for kv cache reuse/chunked context in MLA - self.allocate_context_kv_indptrs(buffers, capture_graph) - self.ctx_uncached_token_indptr = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - cache_name="ctx_uncached_token_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_uncached_token_indptr = torch.zeros_like( - self.ctx_uncached_token_indptr, - device='cpu', - pin_memory=prefer_pinned(), - ) - - def allocate_context_kv_indptrs(self, buffers, capture_graph: bool) -> None: - """Allocate the context cached-token and KV-length prefix buffers. - - ``ctx_cached_token_indptr`` / ``ctx_kv_indptr`` [max_num_requests + 1] - (int64) and their pinned host copies; filled per step by - ``stage_context_cached_kv`` or the context-MLA preparation. - """ - md = self.metadata - self.ctx_cached_token_indptr = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - cache_name="ctx_cached_token_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_cached_token_indptr = torch.zeros_like( - self.ctx_cached_token_indptr, - device='cpu', - pin_memory=prefer_pinned(), - ) - # context full seqlens include cached tokens and uncached tokens - self.ctx_kv_indptr = md.get_empty( - buffers, - (self.max_num_requests + 1, ), - cache_name="ctx_kv_indptr", - dtype=torch.int64, - capture_graph=capture_graph, - ) - self.host_ctx_kv_indptr = torch.zeros_like( - self.ctx_kv_indptr, - device='cpu', - pin_memory=prefer_pinned(), - ) - - def adopt_target_state(self, source: "TrtllmCoreMetadata") -> None: - """Take the target's prepared state at a draft-loop start. - - The runtime KV lengths are the draft's own device lengths; its MLA - prefixes and split plan are its own, built lazily from its own lengths. - """ - super().adopt_target_state(source) - self._invalidate_mla_scheduler_buffers() - num_seqs = self.num_seqs - self.kv_lens_cuda_runtime = self.kv_lens_cuda[:num_seqs] - if getattr(source, "host_request_types_runtime", None) is not None: - self.host_request_types_runtime = self.host_request_types[:num_seqs] - - def derive(self) -> None: - md = self.metadata - self.context_plan = None - # Recomputed on first use this iteration; see mla_prepare_scheduler_buffers. - self._invalidate_mla_scheduler_buffers() - self._steady_block_table_key = None - self.steady_decode_kv_lens = None - self._prepared_full_block_offsets = False - extra_attrs = get_model_extra_attrs() - # If model extra attrs is set, attention_metadata is setup in executor. - if extra_attrs is None: - get_global_attrs().attention_metadata = weakref.ref(md) - if self.kv_cache_manager is None: - # Convert the attention metadata to a TRT-LLM no cache attention metadata. - assert self.kv_cache_manager is None, "no cache attention should not have KV cache manager" - assert self._max_seq_len_storage is not None, "max_seq_len should be set for no cache attention" - - # setting kv cache params - self.kv_cache_params = KVCacheParams(use_cache=False, ) - - # trtllm attn metadata prepare() requires this - self.prompt_lens = self.context_lens - - self.kv_cache_block_offsets = None - self.block_ids_per_seq = None - - prompt_lens = torch.as_tensor( - self.prompt_lens, - dtype=torch.int, - device='cpu', - ) - self.prompt_lens_cpu[:self.num_seqs].copy_(prompt_lens) - copy_h2d_pool_shared(self.prompt_lens_cuda[:self.num_seqs], - self.prompt_lens_cpu[:self.num_seqs], - pack=True) - - # number of tokens in the kv cache for each sequence in the batch - cached_token_lens = torch.tensor( - self.kv_cache_params.num_cached_tokens_per_seq, - dtype=torch.int, - device='cpu', - ) if self.kv_cache_params.use_cache else None - - self._prepare_helix_context_slots(cached_token_lens) - - # number of tokens needed in the kv cache for each sequence after the next pass. - kv_lens = cached_token_lens + \ - self.seq_lens_kv if cached_token_lens is not None else self.seq_lens_kv - if kv_lens is self.seq_lens or kv_lens is self.seq_lens_kv: - kv_lens = kv_lens.clone() - - # self.kv_lens is the valid kv cache length, while the self.kv_lens_cuda is - # the sequence length including the cached tokens and the input tokens. - self.kv_lens[:self.num_seqs].copy_( - kv_lens + self.kv_cache_params.num_extra_kv_tokens) - if self.kv_lens_offsets is not None: - self.kv_lens_cuda[:self.num_seqs].copy_(maybe_pin_memory( - kv_lens[:self.num_seqs]), - non_blocking=True) - # Corrections known only on device; the host copies keep the - # scheduled lengths. - self.kv_lens_cuda[:self.num_seqs] += self.kv_lens_offsets[:self. - num_seqs] - else: - # Nothing on the device touches the lengths before the forward, so - # the copy may join the step-input pack. - step_inputs.copy_(self.kv_lens_cuda[:self.num_seqs], - maybe_pin_memory(kv_lens[:self.num_seqs])) - # total kv lens for context requests and generation requests, without extra tokens - self.host_total_kv_lens[0] = kv_lens[:self.num_contexts].sum().item() - self.host_total_kv_lens[1] = kv_lens[self.num_contexts:self. - num_seqs].sum().item() - self.host_request_types[:self.num_contexts].fill_(0) - self.host_request_types[self.num_contexts:self.num_seqs].fill_(1) - - # prepare for kv cache reuse/chunked context in MLA - if self.enable_context_mla_with_cached_kv: - self.prepare_context_mla_with_cached_kv(cached_token_lens, kv_lens) - - # kv block offsets - assert self.request_ids is not None - if self.kv_cache_manager is not None: - max_kv_len = int(self.kv_lens[:self.num_seqs].max()) - assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( - f"The max KV cache length of input sequences ({max_kv_len}) " - f"exceeds the KV cache manager's maximum supported length " - f"({self.kv_cache_manager.max_seq_len}).") - - # On the non-speculative path the host kv_lens snapshot bounds - # every block-table access, so the staged/H2D width can be capped - # at the batch's maximum instead of max_seq_len's worth of - # columns. Speculative decoding must stage the full width: - # draft/tree sub-steps and the overlap scheduler advance - # kv_lens_cuda on device past the host snapshot, and their - # kernels dereference block columns a host-derived cap would - # leave unstaged (uninitialized in this buffer). - spec_active = (md.is_spec_decoding_enabled - or bool(self.kv_cache_params.num_extra_kv_tokens) or - (self.runtime_features is not None and - self.runtime_features.has_speculative_draft_tokens) - or self.full_block_table_width) - max_blocks = None - if not spec_active and self.kv_cache_manager.tokens_per_block: - max_blocks = ceil_div(max_kv_len, - self.kv_cache_manager.tokens_per_block) - self.kv_cache_manager.prepare_batch_tables(self.request_ids, - self.num_contexts) - self.kv_cache_manager.copy_batch_block_offsets( - self.kv_cache_block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - self.num_seqs, - max_blocks=max_blocks) - self._prepared_full_block_offsets = max_blocks is None - - # Don't pass self.kv_lens as kv_lens here because it includes extra - # tokens. Use the actual KV length (without extra tokens) for - # kv_lens_runtime, which becomes host_past_key_value_lengths and - # eventually mMaxSeqLenKv. - self.bind_runtime_views( - kv_lens_cuda=self.kv_lens_cuda[:self.num_seqs], - kv_lens=kv_lens[:self.num_seqs], - prompt_lens_cuda=self.prompt_lens_cuda[:self.num_seqs], - prompt_lens_cpu=self.prompt_lens_cpu[:self.num_seqs], - host_request_types=self.host_request_types[:self.num_seqs], - ) - - if self._has_mixed_precision_staging(): - # Native BF16 paged-context FMHA reads 32-token pages. - self.prepare_mixed_precision_kv( - "HND", min(self.kv_cache_manager.tokens_per_block, 32)) - if self.kv_lens_offsets is not None: - self._sync_mixed_precision_staging_lengths() - - @property - def effective_beam_width(self) -> int: - # Only use this for the fallback kernel's beam_width argument. - # Cross-attention reads request-scoped encoder K/V that is written once - # and reused unchanged by every decoder beam. Metadata preparation still - # uses beam_width to expand cross block-offset rows to decoder-sequence - # scope, but the fallback kernel should treat the cross K/V cache as - # non-beam-packed. - return 1 if self.is_cross else self.beam_width - - @property - def effective_workspace(self) -> Optional[torch.Tensor]: - """Attention-kernel workspace, switching to the CUDA-graph copy under capture.""" - return self.cuda_graph_workspace if self.is_cuda_graph else self.workspace - - @property - def max_context_length(self) -> int: - """ - Upper bound for a single context window. - Required max_seq_len for context-only attention cases like visual gen - """ - return min(self.max_seq_len, self.max_num_tokens) - - @property - def max_seq_len(self) -> int: - """ - Returns the max sequence length. - If the attention uses KV cache, it will return max_seq_len from the KV cache manager. - If the attention is no cache, max_seq_len should be set manually by user. - """ - if self.kv_cache_manager is not None: - return self.kv_cache_manager.max_seq_len - else: - assert self._max_seq_len_storage is not None, "max_seq_len should be set for no kv cache attention" - return self._max_seq_len_storage - - # Block tables given to a metadata without a KV cache manager; with one, - # the core allocates or derives them from the manager. - _manager_block_tables = frozenset({ - "kv_cache_block_offsets", - "host_kv_cache_block_offsets", - "block_ids_per_seq", - "kv_block_ids_per_seq", - }) - - def creation_value(self, name: str) -> Any: - # The value set for no-cache attention; a cache's comes from its manager. - if name == "max_seq_len": - return self._max_seq_len_storage - if name in self._manager_block_tables and self.kv_cache_manager is not None: - return None - return super().creation_value(name) - - @max_seq_len.setter - def max_seq_len(self, value: int) -> None: - """ - Set the max sequence length for no cache attention. - """ - self._max_seq_len_storage = value - - @property - def tokens_per_block(self) -> Optional[int]: - """ - Returns the number of tokens per block from the KV cache manager. - """ - pools = self._extension_kv_cache_pools() - if pools is not None: - return pools[2] - return self.kv_cache_manager.tokens_per_block if self.kv_cache_manager is not None else None - - @property - def page_size(self) -> int: - """ - Number of tokens per cache page. - """ - page_size_override = getattr(self, '_page_size_override', None) - if page_size_override is not None: - return page_size_override - assert self.kv_cache_manager is not None, "page_size requires a KV cache manager" - return self.kv_cache_manager.tokens_per_block - - @page_size.setter - def page_size(self, value: int) -> None: - self._page_size_override = value - - @property - def host_kv_cache_pool_pointers(self) -> Optional[torch.Tensor]: - """ - Returns the host KV cache pool pointers from the KV cache manager if KV cache manager is not None. - """ - pools = self._extension_kv_cache_pools() - if pools is not None: - return pools[0] - return self.kv_cache_manager.kv_cache_pool_pointers if self.kv_cache_manager is not None else None + if self.cuda_graph_workspace is None: + self.cuda_graph_workspace = torch.empty( + (0, ), + device='cuda', + dtype=torch.int8, + ) - @property - def host_kv_cache_pool_mapping(self) -> Optional[torch.Tensor]: - """ - Returns the host KV cache pool mapping from the KV cache manager if KV cache manager is not None. - """ - pools = self._extension_kv_cache_pools() - if pools is not None: - return pools[1] - return self.kv_cache_manager.kv_cache_pool_mapping if self.kv_cache_manager is not None else None + if self.kv_cache_manager is not None: + num_attention_op_pools = getattr(self.kv_cache_manager, + "num_attention_op_pools", + self.kv_cache_manager.num_pools) + self.kv_cache_block_offsets = md.get_empty( + buffers, + [ + num_attention_op_pools, self.max_num_sequences, 2, + self.kv_cache_manager.max_blocks_per_seq + ], + cache_name="kv_cache_block_offsets", + dtype=torch.int32, + capture_graph=capture_graph, + ) + self.host_kv_cache_block_offsets = self.kv_cache_manager.host_kv_cache_block_offsets + self.block_ids_per_seq = None + self.kv_block_ids_per_seq = None - def _extension_kv_cache_pools( - self) -> Optional[Tuple[torch.Tensor, torch.Tensor, int]]: - # An extension's pools (CSA2 compute tiles) replace the manager's. - return self.metadata.find("kv_cache_pools", None) + if self.enable_context_mla_with_cached_kv: + # for kv cache reuse/chunked context in MLA + self.allocate_context_kv_indptrs(buffers, capture_graph) + self.ctx_uncached_token_indptr = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + cache_name="ctx_uncached_token_indptr", + dtype=torch.int64, + capture_graph=capture_graph, + ) + self.host_ctx_uncached_token_indptr = torch.zeros_like( + self.ctx_uncached_token_indptr, + device='cpu', + pin_memory=prefer_pinned(), + ) - def copy_lengths_from(self, source: "AttentionCoreMetadata") -> None: - super().copy_lengths_from(source) - num_seqs = self.num_seqs - self.kv_lens_cuda[:num_seqs].copy_(source.kv_lens_cuda[:num_seqs]) - if self.host_request_types is not None: - self.host_request_types[:num_seqs].copy_( - source.host_request_types[:num_seqs]) + def allocate_context_kv_indptrs(self, buffers, capture_graph: bool) -> None: + """Allocate the context cached-token and KV-length prefix buffers. - def resync_length_derived_state(self) -> None: - # A draft step changed the lengths the layer-invariant prefixes derive - # from; the next MLA layer rebuilds them, and the next context-only - # call plans again. - self.context_plan = None - self._invalidate_mla_scheduler_buffers() - self._sync_mixed_precision_staging_lengths() + ``ctx_cached_token_indptr`` / ``ctx_kv_indptr`` [max_num_requests + 1] + (int64) and their pinned host copies; filled per step by + ``stage_context_cached_kv`` or the context-MLA preparation. + """ + md = self.metadata + self.ctx_cached_token_indptr = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + cache_name="ctx_cached_token_indptr", + dtype=torch.int64, + capture_graph=capture_graph, + ) + self.host_ctx_cached_token_indptr = torch.zeros_like( + self.ctx_cached_token_indptr, + device='cpu', + pin_memory=prefer_pinned(), + ) + # context full seqlens include cached tokens and uncached tokens + self.ctx_kv_indptr = md.get_empty( + buffers, + (self.max_num_requests + 1, ), + cache_name="ctx_kv_indptr", + dtype=torch.int64, + capture_graph=capture_graph, + ) + self.host_ctx_kv_indptr = torch.zeros_like( + self.ctx_kv_indptr, + device='cpu', + pin_memory=prefer_pinned(), + ) - def _sync_mixed_precision_staging_lengths(self) -> None: - """Keep the mixed-precision staging lengths in step with kv_lens_cuda. + def adopt_target_state(self, source: "TrtllmCoreMetadata") -> None: + """Take the target's prepared state at a draft-loop start. - The staging copy loads the host lengths in ``prepare_mixed_precision_kv``; - the step's device corrections (``kv_lens_offsets``) and a draft loop's - length changes reach only kv_lens_cuda, so the staging lengths are - taken from it on the device. + The runtime KV lengths are the draft's own device lengths; its MLA + prefixes and split plan are its own, built lazily from its own lengths. """ - view = self.mixed_precision_staging - if (view is None or view.lengths is None - or view.manager is not self.kv_cache_manager): - return - lengths = view.lengths - n = lengths.shape[1] - lengths[1].copy_(self.kv_lens_cuda[:n]) - torch.sub(lengths[1], self.seq_lens_kv_cuda[:n], out=lengths[0]) + super().adopt_target_state(source) + self._invalidate_mla_scheduler_buffers() + num_seqs = self.num_seqs + self.kv_lens_cuda_runtime = self.kv_lens_cuda[:num_seqs] + if getattr(source, "host_request_types_runtime", None) is not None: + self.host_request_types_runtime = self.host_request_types[:num_seqs] def _invalidate_mla_scheduler_buffers(self) -> None: # Spec-dec rewrites q_lens and kv_lens between sub-steps, so the cumulative @@ -1379,6 +1244,141 @@ def presented_token_major( self.host_request_types_runtime, self._num_generations, self.max_num_sequences) = saved + def derive(self) -> None: + md = self.metadata + self.context_plan = None + # Recomputed on first use this iteration; see mla_prepare_scheduler_buffers. + self._invalidate_mla_scheduler_buffers() + self._steady_block_table_key = None + self.steady_decode_kv_lens = None + self._prepared_full_block_offsets = False + extra_attrs = get_model_extra_attrs() + # If model extra attrs is set, attention_metadata is setup in executor. + if extra_attrs is None: + get_global_attrs().attention_metadata = weakref.ref(md) + if self.kv_cache_manager is None: + # Convert the attention metadata to a TRT-LLM no cache attention metadata. + assert self.kv_cache_manager is None, "no cache attention should not have KV cache manager" + assert self._max_seq_len_storage is not None, "max_seq_len should be set for no cache attention" + + # setting kv cache params + self.kv_cache_params = KVCacheParams(use_cache=False, ) + + # trtllm attn metadata prepare() requires this + self.prompt_lens = self.context_lens + + self.kv_cache_block_offsets = None + self.block_ids_per_seq = None + + prompt_lens = torch.as_tensor( + self.prompt_lens, + dtype=torch.int, + device='cpu', + ) + self.prompt_lens_cpu[:self.num_seqs].copy_(prompt_lens) + copy_h2d_pool_shared(self.prompt_lens_cuda[:self.num_seqs], + self.prompt_lens_cpu[:self.num_seqs], + pack=True) + + # number of tokens in the kv cache for each sequence in the batch + cached_token_lens = torch.tensor( + self.kv_cache_params.num_cached_tokens_per_seq, + dtype=torch.int, + device='cpu', + ) if self.kv_cache_params.use_cache else None + + self._prepare_helix_context_slots(cached_token_lens) + + # number of tokens needed in the kv cache for each sequence after the next pass. + kv_lens = cached_token_lens + \ + self.seq_lens_kv if cached_token_lens is not None else self.seq_lens_kv + if kv_lens is self.seq_lens or kv_lens is self.seq_lens_kv: + kv_lens = kv_lens.clone() + + # self.kv_lens is the valid kv cache length, while the self.kv_lens_cuda is + # the sequence length including the cached tokens and the input tokens. + self.kv_lens[:self.num_seqs].copy_( + kv_lens + self.kv_cache_params.num_extra_kv_tokens) + if self.kv_lens_offsets is not None: + self.kv_lens_cuda[:self.num_seqs].copy_(maybe_pin_memory( + kv_lens[:self.num_seqs]), + non_blocking=True) + # Corrections known only on device; the host copies keep the + # scheduled lengths. + self.kv_lens_cuda[:self.num_seqs] += self.kv_lens_offsets[:self. + num_seqs] + else: + # Nothing on the device touches the lengths before the forward, so + # the copy may join the step-input pack. + step_inputs.copy_(self.kv_lens_cuda[:self.num_seqs], + maybe_pin_memory(kv_lens[:self.num_seqs])) + # total kv lens for context requests and generation requests, without extra tokens + self.host_total_kv_lens[0] = kv_lens[:self.num_contexts].sum().item() + self.host_total_kv_lens[1] = kv_lens[self.num_contexts:self. + num_seqs].sum().item() + self.host_request_types[:self.num_contexts].fill_(0) + self.host_request_types[self.num_contexts:self.num_seqs].fill_(1) + + # prepare for kv cache reuse/chunked context in MLA + if self.enable_context_mla_with_cached_kv: + self.prepare_context_mla_with_cached_kv(cached_token_lens, kv_lens) + + # kv block offsets + assert self.request_ids is not None + if self.kv_cache_manager is not None: + max_kv_len = int(self.kv_lens[:self.num_seqs].max()) + assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( + f"The max KV cache length of input sequences ({max_kv_len}) " + f"exceeds the KV cache manager's maximum supported length " + f"({self.kv_cache_manager.max_seq_len}).") + + # On the non-speculative path the host kv_lens snapshot bounds + # every block-table access, so the staged/H2D width can be capped + # at the batch's maximum instead of max_seq_len's worth of + # columns. Speculative decoding must stage the full width: + # draft/tree sub-steps and the overlap scheduler advance + # kv_lens_cuda on device past the host snapshot, and their + # kernels dereference block columns a host-derived cap would + # leave unstaged (uninitialized in this buffer). + spec_active = (md.is_spec_decoding_enabled + or bool(self.kv_cache_params.num_extra_kv_tokens) or + (self.runtime_features is not None and + self.runtime_features.has_speculative_draft_tokens) + or self.full_block_table_width) + max_blocks = None + if not spec_active and self.kv_cache_manager.tokens_per_block: + max_blocks = ceil_div(max_kv_len, + self.kv_cache_manager.tokens_per_block) + self.kv_cache_manager.prepare_batch_tables(self.request_ids, + self.num_contexts) + self.kv_cache_manager.copy_batch_block_offsets( + self.kv_cache_block_offsets, + self.request_ids, + self.beam_width, + self.num_contexts, + self.num_seqs, + max_blocks=max_blocks) + self._prepared_full_block_offsets = max_blocks is None + + # Don't pass self.kv_lens as kv_lens here because it includes extra + # tokens. Use the actual KV length (without extra tokens) for + # kv_lens_runtime, which becomes host_past_key_value_lengths and + # eventually mMaxSeqLenKv. + self.bind_runtime_views( + kv_lens_cuda=self.kv_lens_cuda[:self.num_seqs], + kv_lens=kv_lens[:self.num_seqs], + prompt_lens_cuda=self.prompt_lens_cuda[:self.num_seqs], + prompt_lens_cpu=self.prompt_lens_cpu[:self.num_seqs], + host_request_types=self.host_request_types[:self.num_seqs], + ) + + if self._has_mixed_precision_staging(): + # Native BF16 paged-context FMHA reads 32-token pages. + self.prepare_mixed_precision_kv( + "HND", min(self.kv_cache_manager.tokens_per_block, 32)) + if self.kv_lens_offsets is not None: + self._sync_mixed_precision_staging_lengths() + def _on_mixed_precision_kv_prepared(self, view) -> None: """Lay out the staging pages as TRTLLM block offsets for this batch.""" batch = len(view.page_counts) @@ -1444,8 +1444,8 @@ def prepare_encoder_decoder_from_precomputed_lengths( assert md.find("draft_or_none", None) is None assert not md.is_spec_decoding_enabled assert self.kv_cache_params.num_extra_kv_tokens == 0 - assert md.find_ext(FlashMlaMetadata) is None - assert md.find_ext(HelixMetadata) is None + assert not self.enable_flash_mla + assert not self.enable_helix assert not self.enable_context_mla_with_cached_kv assert self.request_ids is not None assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( @@ -2657,8 +2657,7 @@ def forward( token_major_view = (metadata.token_major_view() if forward_args.attention_input_type == AttentionInputType.generation_only else None) - if (metadata.find_ext(FlashMlaMetadata) is not None - and token_major_view is not None): + if metadata.enable_flash_mla and token_major_view is not None: raise ValueError( "FlashMLA does not support ragged token-major generation; " "disable FlashMLA for this layout.") @@ -3077,7 +3076,7 @@ def _build_context_plan(self, q: torch.Tensor, or metadata.spec_decoding_bl_tree_mask is not None or metadata.spec_bl_tree_first_sparse_mask_offset_kv is not None or metadata.helix_position_offsets is not None - or metadata.find_ext(FlashMlaMetadata) is not None): + or metadata.enable_flash_mla): return plan from .fmha.flashinfer_trtllm_gen import FlashInferTrtllmGenFmha fmha = self._fmha_manager.select(self, q, None, None, metadata, fwd) @@ -3354,8 +3353,8 @@ def append_fp4_mla_context_cache( if not self.uses_fp4_mla_attention or self.mla_params is None: raise RuntimeError( "FP4 MLA context append requires dense FP4 MLA attention.") - fp4_mla_state = metadata.find_ext(Fp4MlaState) - if fp4_mla_state is None or fp4_mla_state.positions is None: + if (metadata.fp4_mla_state is None + or metadata.fp4_mla_state.positions is None): raise RuntimeError( "FP4 MLA context append requires token positions.") self._ensure_rope_table_size(metadata.max_seq_len) diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py index 3830e814da03..8ee3395d95f9 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/inputs.py @@ -11,7 +11,6 @@ import torch from tensorrt_llm._device_copy import DeviceCopyBuffer -from tensorrt_llm._torch.attention.backends.features import HelixMetadata from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata, BatchInputs from tensorrt_llm._torch.distributed import Distributed from tensorrt_llm._torch.kernels.spec_step_copies import op as spec_step_copies @@ -2463,7 +2462,7 @@ def _prepare_attn_metadata( # arm the spec path for ordinary helix generation and send its # consumers to uninitialized buffers. helix_spec_active = bool(enable_spec_decode and host.lists.helix_owned_new_tokens) - if attn_metadata.find_ext(HelixMetadata) is not None: + if attn_metadata.enable_helix: attn_metadata.update_helix_param( helix_position_offsets=host.lists.helix_position_offsets, helix_is_inactive_rank=host.lists.helix_is_inactive_rank, diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index 940b336ce611..b3b0f6797709 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -17,7 +17,6 @@ from tensorrt_llm.mapping import Mapping from ..attention.backends import AttentionMetadata -from ..attention.backends.features import FlashMlaMetadata from ..attention.backends.flashinfer import FlashInferAttentionMetadata from ..attention.backends.sparse.params import MTPIndexShareExtension from ..model_config import ModelConfig @@ -1156,8 +1155,7 @@ def _prepare_flash_mla_generation_layout(self, attn_metadata, num_contexts, With no context rows ``prepare_flash_mla`` already staged this exact layout, so the early return merely skips a redundant copy. """ - if (num_contexts <= 0 - or attn_metadata.find_ext(FlashMlaMetadata) is None): + if num_contexts <= 0 or not attn_metadata.enable_flash_mla: return attn_metadata.block_ids_per_seq[:batch_size].copy_( attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True) diff --git a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py index 900a85bede99..6a3de8243fa8 100644 --- a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py +++ b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py @@ -2001,8 +2001,8 @@ def test_on_update_kv_lens_rederives_lengths_slots_and_counts(nvfp4): metadata.msa_cu_kv_lens = torch.tensor([0, 9, 21, -1], dtype=torch.int32) metadata.msa_cu_q_lens = torch.tensor([0, 2, 5], dtype=torch.int32) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() # Corrections must be idempotent. + metadata.on_update_kv_lens() + metadata.on_update_kv_lens() # Corrections must be idempotent. if nvfp4: assert metadata.msa_cu_kv_lens.tolist() == [0, 8, 20, -1] assert metadata.msa_cu_q_lens.tolist() == [0, 2, 5] @@ -2027,7 +2027,7 @@ def test_on_update_kv_lens_clamps_to_the_staged_lens(): qo_lens=(1, 1), kv_staged=(5, 7), kv_corrected=(9, 7), page_size=4 ) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [5, 7] assert metadata.msa_out_cache_loc[:2].tolist() == [4, 106] @@ -2048,7 +2048,7 @@ def test_on_update_kv_lens_patches_a_mixed_steps_per_step_counts(): prefill_counts=True, ) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [5, 11] # Context row: positions 0..4 -> slots 0..4. Generation row attends 11 @@ -2082,7 +2082,7 @@ def test_on_update_kv_lens_is_a_noop_without_speculative_decoding(): metadata._msa_kv_lens_dynamic = False - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() assert metadata.msa_seq_lens_cuda.tolist() == [4, 6] assert metadata.msa_out_cache_loc.tolist() == [-1] * 5 @@ -2153,7 +2153,7 @@ def test_a_taken_kv_update_replaces_the_rederivation(prefill_counts): kv_update = _RecordingKvUpdate(taken=True) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() # The hook owns the update: the metadata's own re-derivation does not run. assert metadata.msa_seq_lens_cuda.tolist() == [9, 12] @@ -2183,7 +2183,7 @@ def test_a_declined_kv_update_falls_back_to_the_rederivation(): kv_update = _RecordingKvUpdate(taken=False) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() assert len(kv_update.calls) == 1 assert metadata.msa_seq_lens_cuda.tolist() == [8, 12] @@ -2199,7 +2199,7 @@ def test_kv_update_is_not_called_without_dynamic_kv_lens(): kv_update = _RecordingKvUpdate(taken=True) metadata.msa_policy = MsaRuntimePolicy(kv_update=kv_update) - metadata.ext(MiniMaxM3MsaMetadata).on_update_kv_lens() + metadata.on_update_kv_lens() assert kv_update.calls == [] From 6b3a611fa7111a89e841f03bc06ba2381d7c9dcc Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 01:42:16 -0800 Subject: [PATCH 3/7] [None][feat] Keep main's layout in the speculative workers and trim new unit tests - The speculative workers whose draft forwards moved into draft_loop() (DFlash, Kimi K3, PARD, draft-target, MTP) keep the moved code at main's indentation in a method called from inside the loop. - Remove the unit tests that only exercised the extension framework itself (extension contract, draft metadata reuse, creation config) and the new tests without a counterpart on main; end-to-end tests cover these paths. Tests that replace main's tests of the retired draft-swap and KV-length-update mechanisms stay. Signed-off-by: Yihan Wang --- .../spec_worker.py | 397 +++++---- .../spec_worker.py | 397 +++++---- .../_torch/attention/backends/flashinfer.py | 2 +- tensorrt_llm/_torch/speculative/dflash.py | 373 ++++---- .../_torch/speculative/draft_target.py | 259 +++--- tensorrt_llm/_torch/speculative/mtp.py | 168 ++-- tensorrt_llm/_torch/speculative/pard.py | 257 +++--- .../test_metadata_extension_contract.py | 812 ------------------ .../sparse/csa2/test_cache_manager.py | 27 - .../attention/sparse/csa2/test_metadata.py | 21 - .../deepseek_v4/test_token_major_rows.py | 49 -- .../qsa/test_qsa_cache_manager_layout.py | 23 - .../sparse/rocketkv/test_rocketkv.py | 70 -- .../attention/test_draft_metadata_reuse.py | 780 ----------------- .../test_metadata_creation_config.py | 176 ---- tests/unittest/_torch/helpers.py | 8 - .../engine/test_decoder_runtime_hooks.py | 6 - .../pyexecutor/test_kv_lens_update_once.py | 6 - tests/unittest/utils/metadata_contract.py | 74 -- 19 files changed, 1024 insertions(+), 2881 deletions(-) delete mode 100644 tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py delete mode 100644 tests/unittest/_torch/attention/test_draft_metadata_reuse.py delete mode 100644 tests/unittest/_torch/attention/test_metadata_creation_config.py delete mode 100644 tests/unittest/utils/metadata_contract.py diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py index 1916a6fa9073..e617d3d33c7a 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp16ep1/spec_worker.py @@ -594,206 +594,243 @@ def _forward_impl( # The drafter's inputs set their lengths on the draft metadata, which leaves the target's untouched. with self.draft_loop(attn_metadata) as draft_attn_metadata: - prep = None - if fused_accept: - accepted_tokens, num_accepted_tokens, prep = self._k3_accept( - logits, - attn_metadata, - spec_metadata, - draft_model, - # The kernel advances the draft's KV lengths: the buffer comes from its core. - draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], - shard, - ) - else: - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata - ) + return self._k3_draft_loop_body( + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + shard, + raw_logits, + K, + draft_kv_cache_manager, + fused_accept, + is_warmup, + ) - # Opt-in acceptance recording (env-gated; eager-mode measurement - # runs only). Skipped for CUDA-graph batches (capture/replay/warmup - # use synthetic requests and forbid the host sync). - if ( - self._accept_stats is not None - and num_gens > 0 - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - ): - self._accept_stats.on_accept( - spec_metadata.request_ids[num_contexts:batch_size], - num_accepted_tokens[num_contexts:batch_size].tolist(), - ) + def _k3_draft_loop_body( + self, + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + shard, + raw_logits, + K, + draft_kv_cache_manager, + fused_accept, + is_warmup, + ): + """The rest of ``_forward_impl``, inside its draft loop.""" + prep = None + if fused_accept: + accepted_tokens, num_accepted_tokens, prep = self._k3_accept( + logits, + attn_metadata, + spec_metadata, + draft_model, + # The kernel advances the draft's KV lengths: the buffer comes from its core. + draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], + shard, + ) + else: + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + logits, attn_metadata, spec_metadata + ) - if fused_accept: - # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft - # metadata re-derives its length-dependent state from them. - self.refresh_draft() - elif num_gens > 0 and isinstance( - attn_metadata.kv_cache_manager, MambaHybridCacheManager - ): - # Update GDN/Mamba recurrent states to the accepted token's state. - attn_metadata.kv_cache_manager.update_mamba_states( - attn_metadata=attn_metadata, - num_accepted_tokens=num_accepted_tokens, - state_indices=attn_metadata.mamba_metadata.state_indices, - ) + # Opt-in acceptance recording (env-gated; eager-mode measurement + # runs only). Skipped for CUDA-graph batches (capture/replay/warmup + # use synthetic requests and forbid the host sync). + if ( + self._accept_stats is not None + and num_gens > 0 + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + ): + self._accept_stats.on_accept( + spec_metadata.request_ids[num_contexts:batch_size], + num_accepted_tokens[num_contexts:batch_size].tolist(), + ) - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) + if fused_accept: + # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft + # metadata re-derives its length-dependent state from them. + self.refresh_draft() + elif num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): + # Update GDN/Mamba recurrent states to the accepted token's state. + attn_metadata.kv_cache_manager.update_mamba_states( + attn_metadata=attn_metadata, + num_accepted_tokens=num_accepted_tokens, + state_indices=attn_metadata.mamba_metadata.state_indices, + ) - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens - ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=draft_attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, - prep=prep, + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] + + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + prep=prep, + ) - if num_gens > 0: - with self.draft_kv_cache_context( - draft_attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), - ) + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_hidden_states = self._draft_block_hidden_states( - draft_model, hidden_states_out, num_gens, block_size, K - ) - gen_logits = self._draft_block_logits( - draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata - ) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_hidden_states = self._draft_block_hidden_states( + draft_model, hidden_states_out, num_gens, block_size, K + ) + gen_logits = self._draft_block_logits( + draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata + ) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with one - # coherent path through the block (absent for plain DFlash). - if getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( - draft_model, - gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - spec_metadata, - ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( + # DFlash 2: replace the independent per-position picks with one + # coherent path through the block (absent for plain DFlash). + if getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( + draft_model, gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], spec_metadata, - batch_size, - num_contexts=num_contexts, ) + vocab_size = gen_logits.shape[-1] - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None here -> no calibration rows collected; the - # DSpark confidence-scheduled verification MR supplies the - # real provider, see accept_stats.py). Guarded off for - # CUDA-graph batches and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], - draft_model, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - gen_draft_tokens, - ) + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, + spec_metadata, + batch_size, + num_contexts=num_contexts, + ) - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs( - spec_metadata, num_contexts, num_gens, K, gen_vocab - ) + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None here -> no calibration rows collected; the + # DSpark confidence-scheduled verification MR supplies the + # real provider, see accept_stats.py). Guarded off for + # CUDA-graph batches and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], + draft_model, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._ctx_len.copy_(self._saved_ctx_len) - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False - - outputs = { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } - if shard is not None: - # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. - outputs["logits_vocab_shard"] = True - return outputs + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._ctx_len.copy_(self._saved_ctx_len) + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False + + outputs = { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } + if shard is not None: + # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. + outputs["logits_vocab_shard"] = True + return outputs def _draft_block_hidden_states( self, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py index 1916a6fa9073..e617d3d33c7a 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/kimi_k3_vl/kimi_k3_mxfp4__sm_100__tp16_moetp4ep4/spec_worker.py @@ -594,206 +594,243 @@ def _forward_impl( # The drafter's inputs set their lengths on the draft metadata, which leaves the target's untouched. with self.draft_loop(attn_metadata) as draft_attn_metadata: - prep = None - if fused_accept: - accepted_tokens, num_accepted_tokens, prep = self._k3_accept( - logits, - attn_metadata, - spec_metadata, - draft_model, - # The kernel advances the draft's KV lengths: the buffer comes from its core. - draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], - shard, - ) - else: - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata - ) + return self._k3_draft_loop_body( + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + shard, + raw_logits, + K, + draft_kv_cache_manager, + fused_accept, + is_warmup, + ) - # Opt-in acceptance recording (env-gated; eager-mode measurement - # runs only). Skipped for CUDA-graph batches (capture/replay/warmup - # use synthetic requests and forbid the host sync). - if ( - self._accept_stats is not None - and num_gens > 0 - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - ): - self._accept_stats.on_accept( - spec_metadata.request_ids[num_contexts:batch_size], - num_accepted_tokens[num_contexts:batch_size].tolist(), - ) + def _k3_draft_loop_body( + self, + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + shard, + raw_logits, + K, + draft_kv_cache_manager, + fused_accept, + is_warmup, + ): + """The rest of ``_forward_impl``, inside its draft loop.""" + prep = None + if fused_accept: + accepted_tokens, num_accepted_tokens, prep = self._k3_accept( + logits, + attn_metadata, + spec_metadata, + draft_model, + # The kernel advances the draft's KV lengths: the buffer comes from its core. + draft_attn_metadata.lengths_for_kernel_update()["kv_lens_cuda"], + shard, + ) + else: + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + logits, attn_metadata, spec_metadata + ) - if fused_accept: - # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft - # metadata re-derives its length-dependent state from them. - self.refresh_draft() - elif num_gens > 0 and isinstance( - attn_metadata.kv_cache_manager, MambaHybridCacheManager - ): - # Update GDN/Mamba recurrent states to the accepted token's state. - attn_metadata.kv_cache_manager.update_mamba_states( - attn_metadata=attn_metadata, - num_accepted_tokens=num_accepted_tokens, - state_indices=attn_metadata.mamba_metadata.state_indices, - ) + # Opt-in acceptance recording (env-gated; eager-mode measurement + # runs only). Skipped for CUDA-graph batches (capture/replay/warmup + # use synthetic requests and forbid the host sync). + if ( + self._accept_stats is not None + and num_gens > 0 + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + ): + self._accept_stats.on_accept( + spec_metadata.request_ids[num_contexts:batch_size], + num_accepted_tokens[num_contexts:batch_size].tolist(), + ) - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) + if fused_accept: + # The kernel recorded the KDA replay acceptance and advanced the draft's KV lengths; the draft + # metadata re-derives its length-dependent state from them. + self.refresh_draft() + elif num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): + # Update GDN/Mamba recurrent states to the accepted token's state. + attn_metadata.kv_cache_manager.update_mamba_states( + attn_metadata=attn_metadata, + num_accepted_tokens=num_accepted_tokens, + state_indices=attn_metadata.mamba_metadata.state_indices, + ) - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens - ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=draft_attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, - prep=prep, + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] + + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + prep=prep, + ) - if num_gens > 0: - with self.draft_kv_cache_context( - draft_attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), - ) + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + **dflash_ctx_rows_kwargs(draft_model, inputs["ctx_rows_start"]), + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_hidden_states = self._draft_block_hidden_states( - draft_model, hidden_states_out, num_gens, block_size, K - ) - gen_logits = self._draft_block_logits( - draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata - ) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_hidden_states = self._draft_block_hidden_states( + draft_model, hidden_states_out, num_gens, block_size, K + ) + gen_logits = self._draft_block_logits( + draft_model, gen_hidden_states, draft_attn_metadata, spec_metadata + ) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with one - # coherent path through the block (absent for plain DFlash). - if getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( - draft_model, - gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - spec_metadata, - ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( + # DFlash 2: replace the independent per-position picks with one + # coherent path through the block (absent for plain DFlash). + if getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( + draft_model, gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], spec_metadata, - batch_size, - num_contexts=num_contexts, ) + vocab_size = gen_logits.shape[-1] - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None here -> no calibration rows collected; the - # DSpark confidence-scheduled verification MR supplies the - # real provider, see accept_stats.py). Guarded off for - # CUDA-graph batches and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], - draft_model, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - gen_draft_tokens, - ) + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, + spec_metadata, + batch_size, + num_contexts=num_contexts, + ) - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs( - spec_metadata, num_contexts, num_gens, K, gen_vocab - ) + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None here -> no calibration rows collected; the + # DSpark confidence-scheduled verification MR supplies the + # real provider, see accept_stats.py). Guarded off for + # CUDA-graph batches and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], + draft_model, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._ctx_len.copy_(self._saved_ctx_len) - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False - - outputs = { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } - if shard is not None: - # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. - outputs["logits_vocab_shard"] = True - return outputs + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._ctx_len.copy_(self._saved_ctx_len) + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False + + outputs = { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } + if shard is not None: + # "logits" is this rank's vocabulary shard: the engine gathers it for any logits post-processor. + outputs["logits_vocab_shard"] = True + return outputs def _draft_block_hidden_states( self, diff --git a/tensorrt_llm/_torch/attention/backends/flashinfer.py b/tensorrt_llm/_torch/attention/backends/flashinfer.py index f2cb2c52914d..fbbfeb2e052f 100644 --- a/tensorrt_llm/_torch/attention/backends/flashinfer.py +++ b/tensorrt_llm/_torch/attention/backends/flashinfer.py @@ -1743,7 +1743,7 @@ def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: assert self.request_ids is not None # start and end indices of each sequence in the ragged key and value - # for md attention it's the same as qo_indptr so avoid computing twice. + # for self attention it's the same as qo_indptr so avoid computing twice. if self.is_cross: # start and end indices of each sequence in the ragged key and value torch.cumsum(self.seq_lens_kv_cuda, diff --git a/tensorrt_llm/_torch/speculative/dflash.py b/tensorrt_llm/_torch/speculative/dflash.py index 5b0c098ab1c3..71c04053e344 100644 --- a/tensorrt_llm/_torch/speculative/dflash.py +++ b/tensorrt_llm/_torch/speculative/dflash.py @@ -2277,194 +2277,233 @@ def _forward_impl( # are saved and restored around the draft. The drafter attends its # own context cache and reads no KV lengths. with self.draft_loop(attn_metadata) as draft_attn_metadata: - # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. - # The draft model uses standard 1D RoPE, so only scalar positions are needed. - if position_ids.ndim == 3: - position_ids = position_ids[0, 0] - else: - position_ids = position_ids.squeeze(0) + return self._dflash_draft_loop_body( + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + raw_logits, + logits_vocab_shard, + K, + draft_kv_cache_manager, + is_warmup, + accepted_tokens, + num_accepted_tokens, + ) + + def _dflash_draft_loop_body( + self, + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + attn_metadata, + spec_metadata, + draft_model, + batch_size, + num_contexts, + num_gens, + raw_logits, + logits_vocab_shard, + K, + draft_kv_cache_manager, + is_warmup, + accepted_tokens, + num_accepted_tokens, + ): + """The rest of ``_forward_impl``, inside its draft loop.""" + # Collapse mrope [3, 1, N] to 1D by taking the first (temporal) dimension. + # The draft model uses standard 1D RoPE, so only scalar positions are needed. + if position_ids.ndim == 3: + position_ids = position_ids[0, 0] + else: + position_ids = position_ids.squeeze(0) - # Get total tokens processed by target model (for hidden state extraction) - total_target_tokens = input_ids.shape[0] + # Get total tokens processed by target model (for hidden state extraction) + total_target_tokens = input_ids.shape[0] - # Capture prefill (context) hidden states for future gen steps. - # This gives the draft model the full prompt context, not just gen tokens. - if num_contexts > 0: - self._store_prefill_context( - draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens - ) - # Rebuild batch_to_slot after prefill assigns new slots - if self._ctx_buf_inited and spec_metadata.request_ids: - self._rebuild_batch_to_slot(spec_metadata.request_ids) - - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=draft_attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - total_target_tokens=total_target_tokens, + # Capture prefill (context) hidden states for future gen steps. + # This gives the draft model the full prompt context, not just gen tokens. + if num_contexts > 0: + self._store_prefill_context( + draft_model, spec_metadata, attn_metadata, position_ids, total_target_tokens ) + # Rebuild batch_to_slot after prefill assigns new slots + if self._ctx_buf_inited and spec_metadata.request_ids: + self._rebuild_batch_to_slot(spec_metadata.request_ids) + + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + total_target_tokens=total_target_tokens, + ) - if num_gens > 0: - with self.draft_kv_cache_context( - draft_attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - # The logits read the metadata the context selects, such as a - # FlashInfer view over a separate draft KV cache. - hidden_states_out = draft_model.dflash_forward( - noise_embedding=inputs["noise_embedding"], - query_positions=inputs["query_positions"], - num_ctx_per_req=inputs["num_ctx_per_req"], - ctx_k_cache=inputs["ctx_k_cache"], - ctx_v_cache=inputs["ctx_v_cache"], - ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], - ctx_kv_cache=inputs["ctx_kv_cache"], - ctx_page_table=inputs["ctx_page_table"], - block_placement=inputs["block_placement"], - ) + if num_gens > 0: + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # The logits read the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + hidden_states_out = draft_model.dflash_forward( + noise_embedding=inputs["noise_embedding"], + query_positions=inputs["query_positions"], + num_ctx_per_req=inputs["num_ctx_per_req"], + ctx_k_cache=inputs["ctx_k_cache"], + ctx_v_cache=inputs["ctx_v_cache"], + ctx_cache_batch_idx=inputs["ctx_cache_batch_idx"], + ctx_kv_cache=inputs["ctx_kv_cache"], + ctx_page_table=inputs["ctx_page_table"], + block_placement=inputs["block_placement"], + ) - # Gather K logits per gen request from the block outputs. - # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. - # Which block slots carry them is a drafter-family convention, - # resolved through _draft_slot_ids. - block_size = self._compute_block_size - gen_gather_ids = self._draft_slot_ids(draft_model, num_gens, block_size, K) - # Shields only the last request: at block_size == K with - # shift_label off, slots run 1..K, so every request reads the - # next one's slot 0 and the last overruns. Degrades, never raises. - gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) - - gen_hidden_states = hidden_states_out[gen_gather_ids] - vocab_parallel = self._vocab_parallel - if ( - vocab_parallel is not None - and vocab_parallel.draft_tokens is not None - and not spec_metadata.wants_advanced_draft_sampling - ): - # Greedy draft tokens straight from this rank's vocab shard - # of the block logits, bit-identical to refining the - # gathered logits and taking the per-position argmax. - local_block = vocab_parallel.head.local_logits(gen_hidden_states) - gen_draft_tokens = vocab_parallel.draft_tokens( - draft_model, local_block.view(num_gens, K, -1), inputs - ) - vocab_size = vocab_parallel.head.vocab_size - else: - gen_logits = draft_model.logits_processor( - gen_hidden_states, draft_model.lm_head, draft_attn_metadata, True - ) - # The drafter shares the target's LM head but not its logits - # processor, so apply the transform its config declares, - # before any refinement: a DSpark Markov bias is added to - # the transformed logits. - gen_logits = draft_model.apply_draft_logit_transform(gen_logits) + # Gather K logits per gen request from the block outputs. + # hidden_states_out is flat: [num_gens * block_size, hidden_dim]. + # Which block slots carry them is a drafter-family convention, + # resolved through _draft_slot_ids. + block_size = self._compute_block_size + gen_gather_ids = self._draft_slot_ids(draft_model, num_gens, block_size, K) + # Shields only the last request: at block_size == K with + # shift_label off, slots run 1..K, so every request reads the + # next one's slot 0 and the last overruns. Degrades, never raises. + gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) + + gen_hidden_states = hidden_states_out[gen_gather_ids] + vocab_parallel = self._vocab_parallel + if ( + vocab_parallel is not None + and vocab_parallel.draft_tokens is not None + and not spec_metadata.wants_advanced_draft_sampling + ): + # Greedy draft tokens straight from this rank's vocab shard + # of the block logits, bit-identical to refining the + # gathered logits and taking the per-position argmax. + local_block = vocab_parallel.head.local_logits(gen_hidden_states) + gen_draft_tokens = vocab_parallel.draft_tokens( + draft_model, local_block.view(num_gens, K, -1), inputs + ) + vocab_size = vocab_parallel.head.vocab_size + else: + gen_logits = draft_model.logits_processor( + gen_hidden_states, draft_model.lm_head, draft_attn_metadata, True + ) + # The drafter shares the target's LM head but not its logits + # processor, so apply the transform its config declares, + # before any refinement: a DSpark Markov bias is added to + # the transformed logits. + gen_logits = draft_model.apply_draft_logit_transform(gen_logits) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_logits = self._refine_block_logits( - draft_model, gen_logits, inputs, spec_metadata - ) + gen_logits = self._refine_block_logits( + draft_model, gen_logits, inputs, spec_metadata + ) - # DFlash 2: replace the independent per-position picks with - # one coherent path through the block (absent for plain - # DFlash). - if getattr(draft_model, "lilicorr", None) is not None: - gen_logits = self._apply_lilicorr( - draft_model, - gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["lilicorr_anchor"], - spec_metadata, - ) - vocab_size = gen_logits.shape[-1] - elif getattr(draft_model, "has_candidate_selector", False): - gen_logits = self._apply_dflash2_selector( - draft_model, - gen_logits, - gen_hidden_states.reshape(num_gens, K, -1), - inputs["first_prev_tokens"], - spec_metadata, - ) - vocab_size = gen_logits.shape[-1] - - gen_draft_tokens = self.sample_draft_tokens( + # DFlash 2: replace the independent per-position picks with + # one coherent path through the block (absent for plain + # DFlash). + if getattr(draft_model, "lilicorr", None) is not None: + gen_logits = self._apply_lilicorr( + draft_model, gen_logits, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["lilicorr_anchor"], spec_metadata, - batch_size, - num_contexts=num_contexts, ) - - # Opt-in confidence-calibration recording (env-gated). The - # confidence values come from an optional provider on the - # recorder (None by default -> no calibration rows collected; - # see accept_stats.py). Guarded off for CUDA-graph batches - # and d2t vocab-mapped drafters. - if ( - self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None - ): - self._accept_stats.record_draft_confidence( - spec_metadata.request_ids[num_contexts:batch_size], + vocab_size = gen_logits.shape[-1] + elif getattr(draft_model, "has_candidate_selector", False): + gen_logits = self._apply_dflash2_selector( draft_model, + gen_logits, gen_hidden_states.reshape(num_gens, K, -1), inputs["first_prev_tokens"], - gen_draft_tokens, + spec_metadata, ) + vocab_size = gen_logits.shape[-1] - else: - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs( - spec_metadata, num_contexts, num_gens, K, gen_vocab - ) + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, + spec_metadata, + batch_size, + num_contexts=num_contexts, + ) - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - else: - next_draft_tokens = gen_draft_tokens + # Opt-in confidence-calibration recording (env-gated). The + # confidence values come from an optional provider on the + # recorder (None by default -> no calibration rows collected; + # see accept_stats.py). Guarded off for CUDA-graph batches + # and d2t vocab-mapped drafters. + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): + self._accept_stats.record_draft_confidence( + spec_metadata.request_ids[num_contexts:batch_size], + draft_model, + gen_hidden_states.reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + else: + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + else: + next_draft_tokens = gen_draft_tokens - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + self._rollback_guided_decoder_after_verify(num_accepted_tokens) - # Restore context lengths after warmup; real runs keep the updates. - if is_warmup: - self._restore_ctx_len_device() - self._restore_ctx_len_host() - self._ctx_len_restore_pending = False + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) + + # Restore context lengths after warmup; real runs keep the updates. + if is_warmup: + self._restore_ctx_len_device() + self._restore_ctx_len_host() + self._ctx_len_restore_pending = False - return { - # The gathered float32 target logits, or this rank's vocab shard - # when the target tokens were selected from it. - "logits": raw_logits, - "logits_vocab_shard": logits_vocab_shard, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + return { + # The gathered float32 target logits, or this rank's vocab shard + # when the target tokens were selected from it. + "logits": raw_logits, + "logits_vocab_shard": logits_vocab_shard, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def _draft_block_width(self, draft_model) -> int: """Block slots the draft forward must compute for K draft tokens. diff --git a/tensorrt_llm/_torch/speculative/draft_target.py b/tensorrt_llm/_torch/speculative/draft_target.py index 8202443a9d4d..b0eb3c5b049d 100644 --- a/tensorrt_llm/_torch/speculative/draft_target.py +++ b/tensorrt_llm/_torch/speculative/draft_target.py @@ -167,127 +167,164 @@ def _forward_impl( # target's untouched; without one, the target's state is saved and # restored around the draft loop. with self.draft_loop(attn_metadata) as draft_attn_metadata: - # Prepare inputs for the first draft forward - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - accepted_tokens=accepted_tokens, - attn_metadata=draft_attn_metadata, - spec_metadata=spec_metadata, + return self._draft_target_loop_body( + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager, + batch_size, + num_contexts, + num_gens, + runtime_draft_len, + raw_logits, + accepted_tokens, + num_accepted_tokens, ) - next_draft_tokens = [] - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - - # Get the draft KV cache manager if using separate layouts - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + def _draft_target_loop_body( + self, + draft_attn_metadata, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager, + batch_size, + num_contexts, + num_gens, + runtime_draft_len, + raw_logits, + accepted_tokens, + num_accepted_tokens, + ): + """The rest of ``_forward_impl``, inside its draft loop.""" + # Prepare inputs for the first draft forward + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + accepted_tokens=accepted_tokens, + attn_metadata=draft_attn_metadata, + spec_metadata=spec_metadata, + ) - with self.draft_kv_cache_context( - draft_attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - inputs["attn_metadata"] = draft_attn_metadata - for i in range(runtime_draft_len): - if i == 0: - start_ids_gen = ( - spec_metadata.batch_indices_cuda[:num_gens] * (runtime_draft_len + 1) - ).long() - gather_ids_gen = ( - start_ids_gen - + num_accepted_tokens[num_contexts:] - - 1 - + draft_attn_metadata.num_ctx_tokens - ) - gather_ids = torch.concat( - [spec_metadata.gather_ids[:num_contexts], gather_ids_gen], dim=0 - ) - else: - gather_ids = spec_metadata.batch_indices_cuda[:batch_size] - - if self.guided_decoder is not None: - new_tokens = inputs["input_ids"][gather_ids] - self.guided_decoder.add_draft_batch( - new_tokens, num_accepted_tokens, draft_step=i - ) - - if original_all_rank_num_tokens is not None: - if i == 0: - draft_attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens - elif spec_metadata.all_rank_num_seqs is not None: - draft_attn_metadata.all_rank_num_tokens = ( - spec_metadata.all_rank_num_seqs - ) - - hidden_states = draft_model.model(**inputs) - if isinstance(hidden_states, tuple): - hidden_states = hidden_states[0] - - # Disable spec-dec mode for chained draft steps - set_use_spec_decoding(draft_attn_metadata, False) - - logits = draft_model.logits_processor( - hidden_states[gather_ids], draft_model.lm_head, draft_attn_metadata, True + next_draft_tokens = [] + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + + # Get the draft KV cache manager if using separate layouts + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + + with self.draft_kv_cache_context( + draft_attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + inputs["attn_metadata"] = draft_attn_metadata + for i in range(runtime_draft_len): + if i == 0: + start_ids_gen = ( + spec_metadata.batch_indices_cuda[:num_gens] * (runtime_draft_len + 1) + ).long() + gather_ids_gen = ( + start_ids_gen + + num_accepted_tokens[num_contexts:] + - 1 + + draft_attn_metadata.num_ctx_tokens ) - if self.guided_decoder is not None: - self.guided_decoder.execute_draft_batch(logits, self._d2t, draft_step=i) + gather_ids = torch.concat( + [spec_metadata.gather_ids[:num_contexts], gather_ids_gen], dim=0 + ) + else: + gather_ids = spec_metadata.batch_indices_cuda[:batch_size] - new_draft_token = self.sample_draft_tokens( - logits, - spec_metadata, - batch_size, - draft_step=i, + if self.guided_decoder is not None: + new_tokens = inputs["input_ids"][gather_ids] + self.guided_decoder.add_draft_batch( + new_tokens, num_accepted_tokens, draft_step=i ) - next_draft_tokens.append(new_draft_token) - # Update inputs and metadata for next draft step - position_ids = inputs["position_ids"][gather_ids] + 1 + if original_all_rank_num_tokens is not None: if i == 0: - draft_attn_metadata.set_seq_lens( - slice(0, batch_size), - host=1, - device=1, - update_counts=True, - ) - if inputs["attn_metadata"].kv_cache_manager is not None: - # The context requests continue as one-token - # generation rows. - draft_attn_metadata.as_generation_rows() - self._update_kv_after_first_draft_step( - draft_attn_metadata, - num_accepted_tokens, - num_contexts, - batch_size, - runtime_draft_len, - ) - else: - self._update_kv_for_chained_draft_step(draft_attn_metadata, batch_size) - - inputs = { - "input_ids": new_draft_token, - "position_ids": position_ids, - "attn_metadata": draft_attn_metadata, - "spec_metadata": spec_metadata, - } - - next_draft_tokens = torch.stack(next_draft_tokens, dim=1) - - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) + draft_attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + elif spec_metadata.all_rank_num_seqs is not None: + draft_attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_seqs + + hidden_states = draft_model.model(**inputs) + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + + # Disable spec-dec mode for chained draft steps + set_use_spec_decoding(draft_attn_metadata, False) + + logits = draft_model.logits_processor( + hidden_states[gather_ids], draft_model.lm_head, draft_attn_metadata, True + ) + if self.guided_decoder is not None: + self.guided_decoder.execute_draft_batch(logits, self._d2t, draft_step=i) + + new_draft_token = self.sample_draft_tokens( + logits, + spec_metadata, + batch_size, + draft_step=i, + ) + next_draft_tokens.append(new_draft_token) + + # Update inputs and metadata for next draft step + position_ids = inputs["position_ids"][gather_ids] + 1 + if i == 0: + draft_attn_metadata.set_seq_lens( + slice(0, batch_size), + host=1, + device=1, + update_counts=True, + ) + if inputs["attn_metadata"].kv_cache_manager is not None: + # The context requests continue as one-token + # generation rows. + draft_attn_metadata.as_generation_rows() + self._update_kv_after_first_draft_step( + draft_attn_metadata, + num_accepted_tokens, + num_contexts, + batch_size, + runtime_draft_len, + ) + else: + self._update_kv_for_chained_draft_step(draft_attn_metadata, batch_size) + + inputs = { + "input_ids": new_draft_token, + "position_ids": position_ids, + "attn_metadata": draft_attn_metadata, + "spec_metadata": spec_metadata, + } + + next_draft_tokens = torch.stack(next_draft_tokens, dim=1) + + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) - set_use_spec_decoding(attn_metadata, True) + set_use_spec_decoding(attn_metadata, True) - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def prepare_1st_drafter_inputs( self, diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index ce6d730916d8..f0ee13b2ef52 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -464,86 +464,98 @@ def _forward_impl( # untouched; without one, the target's state is saved and restored # around them. with self.draft_loop(attn_metadata) as draft_attn_metadata: - self.change_attn_metadata(num_accepted_tokens, - draft_attn_metadata, - spec_metadata, - target=attn_metadata) - - # Run MTP layers to predict draft tokens - next_draft_tokens = [] - last_tokens_idx = torch.cumsum( - draft_attn_metadata.seq_lens_cuda, dim=0, dtype=torch.long) - 1 - - draft_kv_cache_manager = self.get_draft_kv_cache_manager( - resource_manager) - - with self.draft_kv_cache_context( - draft_attn_metadata, - draft_kv_cache_manager) as draft_attn_metadata: - # Forward on the metadata the context selects, such as a - # FlashInfer view over a separate draft KV cache. - draft_inputs["attn_metadata"] = draft_attn_metadata - for i, mtp_layer in enumerate( - draft_model.mtp_layers[:runtime_draft_len]): - if self.guided_decoder is not None: - new_tokens = draft_inputs['input_ids'][last_tokens_idx] - self.guided_decoder.add_draft_batch(new_tokens, - num_accepted_tokens, + return self._mtp_draft_loop_body( + draft_attn_metadata, input_ids, hidden_states, logits, + attn_metadata, spec_metadata, draft_model, resource_manager, + runtime_draft_len, batch_size, raw_logits, accepted_tokens, + num_accepted_tokens, draft_inputs) + + def _mtp_draft_loop_body(self, draft_attn_metadata, input_ids, + hidden_states, logits, attn_metadata, + spec_metadata, draft_model, resource_manager, + runtime_draft_len, batch_size, raw_logits, + accepted_tokens, num_accepted_tokens, + draft_inputs): + """The rest of ``_forward_impl``, inside its draft loop.""" + self.change_attn_metadata(num_accepted_tokens, + draft_attn_metadata, + spec_metadata, + target=attn_metadata) + + # Run MTP layers to predict draft tokens + next_draft_tokens = [] + last_tokens_idx = torch.cumsum( + draft_attn_metadata.seq_lens_cuda, dim=0, dtype=torch.long) - 1 + + draft_kv_cache_manager = self.get_draft_kv_cache_manager( + resource_manager) + + with self.draft_kv_cache_context( + draft_attn_metadata, + draft_kv_cache_manager) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + draft_inputs["attn_metadata"] = draft_attn_metadata + for i, mtp_layer in enumerate( + draft_model.mtp_layers[:runtime_draft_len]): + if self.guided_decoder is not None: + new_tokens = draft_inputs['input_ids'][last_tokens_idx] + self.guided_decoder.add_draft_batch(new_tokens, + num_accepted_tokens, + draft_step=i) + + hidden_states = mtp_layer(embed_tokens=draft_model.embed_tokens, + **draft_inputs) + + logits = mtp_layer.shared_head(hidden_states, + draft_model.lm_head, + draft_attn_metadata).float() + if self.guided_decoder is not None: + self.guided_decoder.execute_draft_batch(logits, draft_step=i) - hidden_states = mtp_layer( - embed_tokens=draft_model.embed_tokens, **draft_inputs) - - logits = mtp_layer.shared_head(hidden_states, - draft_model.lm_head, - draft_attn_metadata).float() - if self.guided_decoder is not None: - self.guided_decoder.execute_draft_batch(logits, - draft_step=i) - - new_draft_token = self.sample_draft_tokens(logits, - spec_metadata, - batch_size, - draft_step=i) - next_draft_tokens.append(new_draft_token) - # shift input_ids and hidden_states - input_ids = draft_inputs["input_ids"] - input_ids[:-1] = input_ids[1:].clone() - input_ids[last_tokens_idx] = new_draft_token - draft_hidden_states = draft_inputs["hidden_states"] - draft_hidden_states[:-1] = draft_hidden_states[1:].clone() - draft_hidden_states[last_tokens_idx] = hidden_states[ - last_tokens_idx, :] - draft_inputs = { - "input_ids": input_ids, - "position_ids": draft_inputs["position_ids"], - "hidden_states": draft_hidden_states, - "attn_metadata": draft_inputs["attn_metadata"], - } - next_draft_tokens = torch.stack(next_draft_tokens, dim=1) - - # Override with SA draft tokens after all MTP layers have run, - # so that MTP layers never see SA tokens in their inputs. - if self.sa_enhancer is not None: - num_contexts = attn_metadata.num_contexts - gen_draft_tokens = next_draft_tokens[num_contexts:] - gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( - gen_draft_tokens) - next_draft_tokens[num_contexts:] = gen_draft_tokens - - # prepare next new tokens to support overlap scheduler - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, next_draft_tokens, - spec_metadata.batch_indices_cuda, batch_size, - num_accepted_tokens) - - return { - 'logits': raw_logits, - 'new_tokens': accepted_tokens, - 'new_tokens_lens': num_accepted_tokens, - 'next_draft_tokens': next_draft_tokens, - 'next_new_tokens': next_new_tokens - } + new_draft_token = self.sample_draft_tokens(logits, + spec_metadata, + batch_size, + draft_step=i) + next_draft_tokens.append(new_draft_token) + # shift input_ids and hidden_states + input_ids = draft_inputs["input_ids"] + input_ids[:-1] = input_ids[1:].clone() + input_ids[last_tokens_idx] = new_draft_token + draft_hidden_states = draft_inputs["hidden_states"] + draft_hidden_states[:-1] = draft_hidden_states[1:].clone() + draft_hidden_states[last_tokens_idx] = hidden_states[ + last_tokens_idx, :] + draft_inputs = { + "input_ids": input_ids, + "position_ids": draft_inputs["position_ids"], + "hidden_states": draft_hidden_states, + "attn_metadata": draft_inputs["attn_metadata"], + } + next_draft_tokens = torch.stack(next_draft_tokens, dim=1) + + # Override with SA draft tokens after all MTP layers have run, + # so that MTP layers never see SA tokens in their inputs. + if self.sa_enhancer is not None: + num_contexts = attn_metadata.num_contexts + gen_draft_tokens = next_draft_tokens[num_contexts:] + gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( + gen_draft_tokens) + next_draft_tokens[num_contexts:] = gen_draft_tokens + + # prepare next new tokens to support overlap scheduler + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, next_draft_tokens, + spec_metadata.batch_indices_cuda, batch_size, num_accepted_tokens) + + return { + 'logits': raw_logits, + 'new_tokens': accepted_tokens, + 'new_tokens_lens': num_accepted_tokens, + 'next_draft_tokens': next_draft_tokens, + 'next_new_tokens': next_new_tokens + } def skip_forward( self, diff --git a/tensorrt_llm/_torch/speculative/pard.py b/tensorrt_llm/_torch/speculative/pard.py index 37c52d396e04..a5a8f0548fb2 100644 --- a/tensorrt_llm/_torch/speculative/pard.py +++ b/tensorrt_llm/_torch/speculative/pard.py @@ -184,131 +184,164 @@ def _forward_impl( # restored around it. target_attn_metadata = attn_metadata with self.draft_loop(target_attn_metadata) as attn_metadata: - self._prepare_kv_for_draft_forward(attn_metadata, num_contexts, batch_size) - - position_ids = position_ids.squeeze(0) - inputs = self.prepare_1st_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, + return self._pard_draft_loop_body( + attn_metadata, + input_ids, + position_ids, + hidden_states, + spec_metadata, + draft_model, + resource_manager, + batch_size, + num_contexts, + num_gens, + raw_logits, + K, + accepted_tokens, + num_accepted_tokens, + sa_manager, ) - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - - if num_gens > 0: - with self.draft_kv_cache_context( - attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - # Forward on the metadata the context selects, such as a - # FlashInfer view over a separate draft KV cache. - inputs["attn_metadata"] = draft_attn_metadata - hidden_states_out = draft_model.model(**inputs) - - # Gather K logits per gen request starting at the bonus position. - # Layout: [acc_0..acc_M, masks] (2K total). Positions M..M+K-1 - # produce K unique draft predictions (bonus + K-1 masks). - gen_start_idx = draft_attn_metadata.num_ctx_tokens - - request_bases = ( - torch.arange(num_gens, dtype=torch.long, device="cuda") * (2 * K) - + gen_start_idx - ) - - gen_num_accepted = num_accepted_tokens[num_contexts:batch_size].long() - base_offsets = gen_num_accepted - 1 # M = bonus position - offsets = torch.arange(K, dtype=torch.long, device="cuda") - - gen_gather_ids = ( - request_bases.unsqueeze(1) - + base_offsets.unsqueeze(1) - + offsets.unsqueeze(0) - ).flatten() - gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) - - gen_logits = draft_model.logits_processor( - hidden_states_out[gen_gather_ids], - draft_model.lm_head, - draft_attn_metadata, - True, - ) + def _pard_draft_loop_body( + self, + attn_metadata, + input_ids, + position_ids, + hidden_states, + spec_metadata, + draft_model, + resource_manager, + batch_size, + num_contexts, + num_gens, + raw_logits, + K, + accepted_tokens, + num_accepted_tokens, + sa_manager, + ): + """The rest of ``_forward_impl``, inside its draft loop.""" + self._prepare_kv_for_draft_forward(attn_metadata, num_contexts, batch_size) + + position_ids = position_ids.squeeze(0) + inputs = self.prepare_1st_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + ) - vocab_size = gen_logits.shape[-1] - gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) + if num_gens > 0: + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata + hidden_states_out = draft_model.model(**inputs) + + # Gather K logits per gen request starting at the bonus position. + # Layout: [acc_0..acc_M, masks] (2K total). Positions M..M+K-1 + # produce K unique draft predictions (bonus + K-1 masks). + gen_start_idx = draft_attn_metadata.num_ctx_tokens + + request_bases = ( + torch.arange(num_gens, dtype=torch.long, device="cuda") * (2 * K) + + gen_start_idx + ) - if self.sa_enhancer is not None and sa_manager is not None: - gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( - gen_draft_tokens - ) - - # Pad from (num_gens, K) to (num_gens, 2K-1). - if K > 1: - pad = torch.zeros((num_gens, K - 1), dtype=torch.int32, device="cuda") - gen_draft_tokens = torch.cat([gen_draft_tokens, pad], dim=1) - - elif num_contexts > 0 and self.use_separate_draft_kv_cache: - # Pure context batch: populate the draft KV cache so it's - # ready when generation starts. - with self.draft_kv_cache_context( - attn_metadata, draft_kv_cache_manager - ) as draft_attn_metadata: - # Forward on the metadata the context selects, such as a - # FlashInfer view over a separate draft KV cache. - inputs["attn_metadata"] = draft_attn_metadata - draft_model.model(**inputs) - gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") + gen_num_accepted = num_accepted_tokens[num_contexts:batch_size].long() + base_offsets = gen_num_accepted - 1 # M = bonus position + offsets = torch.arange(K, dtype=torch.long, device="cuda") - else: - gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs( - spec_metadata, num_contexts, num_gens, K, gen_vocab - ) + gen_gather_ids = ( + request_bases.unsqueeze(1) + base_offsets.unsqueeze(1) + offsets.unsqueeze(0) + ).flatten() + gen_gather_ids = gen_gather_ids.clamp(max=hidden_states_out.shape[0] - 1) - if num_contexts > 0 and num_gens > 0: - ctx_draft_tokens = torch.zeros( - (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" + gen_logits = draft_model.logits_processor( + hidden_states_out[gen_gather_ids], + draft_model.lm_head, + draft_attn_metadata, + True, ) - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - elif num_contexts > 0: - next_draft_tokens = torch.zeros( - (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" + + vocab_size = gen_logits.shape[-1] + gen_logits = gen_logits.reshape(num_gens, K, vocab_size) + + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, + spec_metadata, + batch_size, + num_contexts=num_contexts, ) - else: - next_draft_tokens = gen_draft_tokens - self._rollback_guided_decoder_after_verify(num_accepted_tokens) + if self.sa_enhancer is not None and sa_manager is not None: + gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( + gen_draft_tokens + ) - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, + # Pad from (num_gens, K) to (num_gens, 2K-1). + if K > 1: + pad = torch.zeros((num_gens, K - 1), dtype=torch.int32, device="cuda") + gen_draft_tokens = torch.cat([gen_draft_tokens, pad], dim=1) + + elif num_contexts > 0 and self.use_separate_draft_kv_cache: + # Pure context batch: populate the draft KV cache so it's + # ready when generation starts. + with self.draft_kv_cache_context( + attn_metadata, draft_kv_cache_manager + ) as draft_attn_metadata: + # Forward on the metadata the context selects, such as a + # FlashInfer view over a separate draft KV cache. + inputs["attn_metadata"] = draft_attn_metadata + draft_model.model(**inputs) + gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") + + else: + gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a one-hot placeholder so + # they are a legal distribution when they become gen requests next iter. + gen_vocab = vocab_size if num_gens > 0 else None + self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + + if num_contexts > 0 and num_gens > 0: + ctx_draft_tokens = torch.zeros( + (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" + ) + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + elif num_contexts > 0: + next_draft_tokens = torch.zeros( + (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" ) + else: + next_draft_tokens = gen_draft_tokens - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } + self._rollback_guided_decoder_after_verify(num_accepted_tokens) + + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) + + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } def _reshape_draft_tokens_for_accept(self, spec_metadata, num_gens, device): # draft_tokens buffer has (2K-1) entries per gen request; extract the K real drafts. diff --git a/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py b/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py deleted file mode 100644 index 7af36ba725c0..000000000000 --- a/tests/unittest/_torch/attention/hw_agnostic/test_metadata_extension_contract.py +++ /dev/null @@ -1,812 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""CPU coverage of the metadata extension naming and attribute contracts.""" - -import copy -import inspect -from types import SimpleNamespace - -import pytest -import torch -from _torch.helpers import refresh_from_lengths, start_draft_loop - -from tensorrt_llm._torch.attention.backends.extensions import ( - MetadataExtension, - reserved_extension_names, -) -from tensorrt_llm._torch.attention.backends.features import ( - HelixMetadata, - RecurrentConfig, - RecurrentMetadata, -) -from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingMetadata -from tensorrt_llm._torch.attention.backends.interface import AttentionMetadata, BatchInputs -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata -from tensorrt_llm._torch.attention.backends.sparse.params import ( - MTPIndexShareExtension, - MTPIndexShareUser, - SparseAlgorithmExtension, -) -from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAMetadata -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata -from tensorrt_llm._torch.memory_buffer_utils import Buffers -from tensorrt_llm._torch.metadata import KVCacheParams - -pytestmark = pytest.mark.cpu_only - - -def _metadata(*extension_types) -> AttentionMetadata: - return AttentionMetadata(max_num_requests=1, max_num_tokens=1, extension_types=extension_types) - - -def test_undeclared_metadata_attribute_is_rejected(): - metadata = _metadata() - with pytest.raises(AttributeError, match="csa2_remote_tail_mode"): - metadata.csa2_remote_tail_mode = "source" - - -def test_forwarded_fields_write_through_to_their_owner(): - metadata = _metadata(_Probe) - metadata.all_rank_num_tokens = [1] - metadata.probe_value = 1 - assert metadata.core().all_rank_num_tokens == [1] - assert metadata.ext(_Probe).probe_value == 1 - metadata._num_tokens = 3 - assert metadata.core()._num_tokens == 3 - - -def test_metadata_built_without_init_routes_core_fields_to_its_core(): - metadata = object.__new__(AttentionMetadata) - assert metadata.request_ids is None - metadata.request_ids = [0] - assert metadata.core().request_ids == [0] - with pytest.raises(AttributeError, match="ad_hoc"): - metadata.ad_hoc = 1 - - -def test_extension_cannot_declare_a_reserved_name(): - assert "prepare" in reserved_extension_names() - with pytest.raises(TypeError, match="reserved"): - - class _Reserved(MetadataExtension): - methods = frozenset({"prepare"}) - - -def test_two_extension_classes_cannot_declare_one_name(): - with pytest.raises(TypeError, match="already declares"): - - class _Shadow(MetadataExtension): - fields = frozenset({"helix_position_offsets"}) - - -def test_subclass_may_redeclare_its_base_names(): - class _Derived(HelixMetadata): - fields = frozenset({"helix_position_offsets"}) - - assert "helix_position_offsets" in _Derived.declared_fields() - - -def test_extension_cannot_shadow_a_core_attribute(): - class _Core(MetadataExtension): - fields = frozenset({"_core_shadow_probe"}) - - class _Metadata(AttentionMetadata): - _instance_attrs = frozenset({"_core_shadow_probe"}) - - with pytest.raises(ValueError, match="already defines"): - _Metadata(max_num_requests=1, max_num_tokens=1, extension_types=(_Core,)) - - -def test_extensions_with_shared_bases_attach_together(): - assert not HelixMetadata.declared_names() & SpecDecodingMetadata.declared_names() - shared_bases = MTPIndexShareUser.declared_names() | SparseAlgorithmExtension.declared_names() - assert not DSAMetadata.declared_names() & QSAMetadata.declared_names() - shared_bases - - -def test_index_share_owns_the_shared_draft_loop_state(): - assert MTPIndexShareExtension in DSAMetadata.requires - assert MTPIndexShareExtension in QSAMetadata.requires - shared = MTPIndexShareExtension.declared_fields() - assert not shared & DSAMetadata.declared_fields() - assert not shared & QSAMetadata.declared_fields() - - -needs_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="seq_lens are staged to CUDA") - - -@needs_cuda -def test_prepare_applies_batch_inputs(): - metadata = _metadata() - metadata.prepare( - BatchInputs( - seq_lens=torch.tensor([3, 1], dtype=torch.int32), - num_contexts=1, - request_ids=[7, 8], - prompt_lens=[3, 5], - beam_width=1, - num_chunked_ctx_requests=0, - ) - ) - assert metadata.num_contexts == 1 - assert metadata.num_generations == 1 - assert metadata.num_tokens == 4 - assert metadata.request_ids == [7, 8] - assert metadata.cu_q_seqlens is None - - -@needs_cuda -def test_metadata_without_explicit_offsets_rejects_gapped_offsets(): - inputs = dict( - seq_lens=torch.tensor([3, 1], dtype=torch.int32), - num_contexts=2, - request_ids=[0, 1], - prompt_lens=[3, 1], - ) - metadata = _metadata() - assert not metadata.supports_explicit_offsets - metadata.prepare(BatchInputs(**inputs, q_offsets=torch.tensor([0, 3, 4], dtype=torch.int32))) - with pytest.raises(ValueError, match="cu_q_seqlens"): - metadata.prepare( - BatchInputs(**inputs, q_offsets=torch.tensor([0, 4, 6], dtype=torch.int32)) - ) - - -@needs_cuda -def test_length_mutation_apis_keep_buffers_and_counts(): - metadata = _metadata() - metadata.prepare( - BatchInputs( - seq_lens=torch.tensor([2, 4, 4], dtype=torch.int32), - num_contexts=1, - request_ids=[0, 1, 2], - prompt_lens=[2, 9, 9], - ) - ) - host, device = metadata.seq_lens, metadata.seq_lens_cuda - if device.is_cuda: - # prepare copies the pinned host lengths to the device asynchronously; - # the host edits below must not race that copy. - torch.cuda.current_stream().synchronize() - metadata.adjust_seq_lens(slice(1, 3), -1) - assert metadata.seq_lens is host and metadata.seq_lens_cuda is device - assert host.tolist() == [2, 3, 3] and device.tolist() == [2, 3, 3] - assert metadata.num_tokens == 8 - - metadata.set_seq_lens(slice(1, 3), host=5, device=5) - assert host.tolist() == [2, 5, 5] and device.tolist() == [2, 5, 5] - assert metadata.num_tokens == 8 - metadata.set_seq_lens(slice(1, 3), host=1, device=1, update_counts=True) - assert metadata.num_tokens == 4 - - -def test_recurrent_settings_come_from_construction(): - metadata = SimpleNamespace(cuda_graph_buffers=None) - assert RecurrentMetadata(metadata).chunk_size == 128 - recurrent = RecurrentMetadata(metadata, recurrent_config=RecurrentConfig(chunk_size=64)) - assert recurrent.chunk_size == 64 - assert recurrent.mamba_metadata_cls is None - - -class _Probe(MetadataExtension): - fields = frozenset({"probe_value"}) - methods = frozenset({"probe"}) - - def __init__(self, metadata, *, source=None, **config) -> None: - super().__init__(metadata, source=source, **config) - self.probe_value = None - - def probe(self) -> str: - return "probe" - - -def test_composed_class_forwards_extension_fields_and_methods(): - metadata = _metadata(_Probe) - probe = metadata.ext(_Probe) - assert isinstance(metadata, AttentionMetadata) - assert metadata.core_class() is AttentionMetadata - assert type(metadata) is not AttentionMetadata - - marker = object() - probe.probe_value = marker - assert probe.probe_value is marker - assert metadata.probe_value is marker - assert metadata.probe() == "probe" - - # Without the extension, the name is not an attribute of the metadata. - plain = _metadata() - assert not hasattr(plain, "probe_value") - with pytest.raises(AttributeError): - plain.probe_value = marker - - -def test_composed_class_is_shared_and_survives_copies(): - first, second = _metadata(_Probe), _metadata(_Probe) - assert type(first) is type(second) - clone = copy.copy(first) - assert type(clone) is type(first) - - -def test_extension_mutation_recomposes(): - metadata = _metadata() - metadata.add_ext(_Probe) - assert metadata.ext(_Probe).probe_value is metadata.probe_value - metadata.remove_ext(_Probe) - assert not hasattr(metadata, "probe_value") - - -class _TensorProbe(MetadataExtension): - fields = frozenset({"probe_scale"}) - - def __init__(self, metadata, *, source=None, **config) -> None: - super().__init__(metadata, source=source, **config) - self.probe_scale = torch.full((1,), 2.0) - - -def _prepared_probe_metadata(seq_lens) -> AttentionMetadata: - metadata = _metadata(_TensorProbe) - metadata.prepare( - BatchInputs( - seq_lens=torch.tensor(seq_lens, dtype=torch.int32), - num_contexts=1, - request_ids=list(range(len(seq_lens))), - prompt_lens=list(seq_lens), - ) - ) - return metadata - - -def _read_forwarded(x: torch.Tensor, metadata: AttentionMetadata) -> torch.Tensor: - out = x * metadata.num_tokens + metadata.num_contexts - out = out + metadata.seq_lens.sum() - return out * metadata.probe_scale - - -@needs_cuda -def test_dynamo_traces_forwarded_fields_without_breaks_or_recompiles(): - from torch._dynamo.testing import CompileCounter - - torch._dynamo.reset() - first = _prepared_probe_metadata([3, 1]) - x = torch.ones(4) - expected = _read_forwarded(x, first) - - explanation = torch._dynamo.explain(_read_forwarded)(x, first) - assert explanation.graph_break_count == 0, explanation.break_reasons - - fullgraph = torch.compile(_read_forwarded, backend="eager", fullgraph=True) - torch.testing.assert_close(fullgraph(x, first), expected) - - torch._dynamo.reset() - counter = CompileCounter() - compiled = torch.compile(_read_forwarded, backend=counter) - torch.testing.assert_close(compiled(x, first), expected) - # Another metadata of the same composed class and batch shape reuses the - # compiled frame; a changed extension tensor value does not recompile. - second = _prepared_probe_metadata([3, 1]) - second.ext(_TensorProbe).probe_scale = torch.full((1,), 3.0) - torch.testing.assert_close(compiled(x, second), _read_forwarded(x, second)) - assert counter.frame_count == 1 - - -def test_core_is_an_extension_attached_first(): - from tensorrt_llm._torch.attention.backends.core import AttentionCoreMetadata - - metadata = _metadata(_Probe) - extensions = metadata.extensions - assert isinstance(extensions[0], AttentionCoreMetadata) - assert isinstance(extensions[0], MetadataExtension) - assert metadata.core() is extensions[0] - assert type(metadata.core()) is metadata.core_metadata_type - - with pytest.raises(ValueError, match="core"): - metadata.remove_ext(AttentionCoreMetadata) - with pytest.raises(ValueError, match="before"): - metadata._extensions.attach_core(metadata.core_metadata_type(metadata)) - - -def test_extension_cannot_attach_before_the_core(): - from tensorrt_llm._torch.attention.backends.extensions import ExtensionRegistry - - registry = ExtensionRegistry() - with pytest.raises(ValueError, match="core"): - registry.attach(object.__new__(AttentionMetadata), (_Probe,)) - - -def test_core_state_is_forwarded_and_routed_without_init(): - metadata = _metadata() - assert metadata.num_contexts == 0 and metadata.beam_width == 1 - metadata.core().beam_width = 4 - assert metadata.beam_width == 4 - assert "beam_width" not in vars(metadata) - - # Metadata built without __init__ attaches its core on first use. - bare = object.__new__(AttentionMetadata) - bare.core().request_ids = [3] - assert bare.request_ids == [3] - assert bare.extensions[0] is bare.core() - - -def test_constructor_routes_core_arguments_and_rejects_unknown_ones(): - metadata = AttentionMetadata( - max_num_requests=2, max_num_tokens=4, request_ids=[1], beam_width=2 - ) - assert metadata.core().request_ids == [1] - assert metadata.beam_width == 2 - with pytest.raises(TypeError, match="no_such_field"): - AttentionMetadata(max_num_requests=2, max_num_tokens=4, no_such_field=1) - - -def test_copy_owns_its_core_state(): - metadata = _metadata() - metadata.core().request_ids = [1] - clone = copy.copy(metadata) - clone.core().request_ids = [2] - assert metadata.request_ids == [1] - assert clone.core() is not metadata.core() - - -def test_constructor_routes_each_argument_to_its_owner(): - from tensorrt_llm.mapping import Mapping - - mapping = Mapping() - metadata = AttentionMetadata( - max_num_requests=2, max_num_tokens=4, mapping=mapping, draft_kv_cache_manager=None - ) - from tensorrt_llm._torch.attention.backends.features import DraftMetadata - - assert metadata.core().mapping is mapping - assert metadata.ext(DraftMetadata).draft_kv_cache_manager is None - # SpecDecodingMetadata takes max_num_heads_per_kv but this family does not attach it. - with pytest.raises(TypeError, match="SpecDecodingMetadata"): - AttentionMetadata(max_num_requests=2, max_num_tokens=4, max_num_heads_per_kv=4) - # None means not given, even for an owner that is not attached. - AttentionMetadata(max_num_requests=2, max_num_tokens=4, max_num_heads_per_kv=None) - with pytest.raises(TypeError, match="max_num_tokens"): - AttentionMetadata(max_num_requests=2) - - -def test_cross_attention_state_attaches_only_with_cross_attention(): - from tensorrt_llm._torch.attention.backends.features import CrossAttentionMetadata - - metadata = _metadata() - assert metadata.find_ext(CrossAttentionMetadata) is None - assert not metadata.has_cross_sub_metadata - assert not hasattr(metadata, "cross") - - -class _RefreshProbe(MetadataExtension): - def __init__(self, metadata, *, source=None, **config) -> None: - super().__init__(metadata, source=source, **config) - self.refreshes = [] - - def refresh_from_draft_lengths(self) -> None: - super().refresh_from_draft_lengths() - self.refreshes.append("refresh_from_draft_lengths") - - -def test_draft_refresh_calls_every_extension_directly(): - metadata = _metadata(_RefreshProbe) - metadata.prepare( - BatchInputs( - seq_lens=torch.tensor([1, 1], dtype=torch.int32), - num_contexts=0, - request_ids=[0, 1], - prompt_lens=[4, 4], - ) - ) - refreshes = metadata.ext(_RefreshProbe).refreshes - refresh_from_lengths(metadata) - assert refreshes == ["refresh_from_draft_lengths"] - - -def test_refresh_draft_runs_only_in_a_draft_loop_and_only_after_changes(): - target = AttentionMetadata( - max_num_requests=2, max_num_tokens=4, one_model_draft=True, extension_types=(_RefreshProbe,) - ) - target.prepare(_draft_batch([1, 1], 0)) - draft_refreshes = target.draft.ext(_RefreshProbe).refreshes - with pytest.raises(RuntimeError, match="draft_loop"): - target.refresh_draft() - with target.draft_loop() as draft: - assert draft is target.draft and target.in_draft_loop - assert not draft.lengths_changed - # Nothing changed: no refresh. - target.refresh_draft() - assert draft_refreshes == [] - draft.set_seq_lens(slice(0, 2), host=2, device=2, update_counts=True) - assert draft.lengths_changed and not target.lengths_changed - with pytest.raises(RuntimeError, match="changed in place"): - draft.check_lengths_current() - target.refresh_draft() - assert draft_refreshes == ["refresh_from_draft_lengths"] - assert not draft.lengths_changed - draft.check_lengths_current() - assert not target.in_draft_loop - # The target's lengths never changed. - assert target.seq_lens.tolist() == [1, 1] - - -def test_lengths_changed_follows_the_length_primitives(): - metadata = _metadata() - metadata.prepare( - BatchInputs( - seq_lens=torch.tensor([2, 2], dtype=torch.int32), - num_contexts=0, - request_ids=[0, 1], - prompt_lens=[4, 4], - ) - ) - assert not metadata.lengths_changed - metadata.check_lengths_current() - metadata.set_seq_lens(slice(0, 2), host=1, device=1, update_counts=True) - assert metadata.lengths_changed - with pytest.raises(RuntimeError, match="changed in place"): - metadata.check_lengths_current() - refresh_from_lengths(metadata) - assert not metadata.lengths_changed - metadata.as_generation_rows() - assert metadata.lengths_changed - metadata.clear_lengths_changed() - assert not metadata.lengths_changed - buffers = metadata.lengths_for_kernel_update() - assert buffers["seq_lens_cuda"] is metadata.seq_lens_cuda - assert metadata.lengths_changed - - -def test_cross_sub_metadata_takes_lengths_from_its_own_inputs(): - decoder = _metadata() - decoder.prepare( - BatchInputs( - seq_lens=torch.tensor([3, 1], dtype=torch.int32), - num_contexts=1, - request_ids=[0, 1], - prompt_lens=[3, 5], - ) - ) - cross = decoder.create_cross_metadata([7, 0]) - assert cross.is_cross - assert cross.seq_lens is decoder.seq_lens - assert cross.seq_lens_cuda is decoder.seq_lens_cuda - assert cross.seq_lens_kv.tolist() == [7, 0] - assert cross.request_ids == [0, 1] and cross.num_contexts == 1 - - # A new decoder batch reaches the sub-metadata only through its own inputs. - decoder.prepare( - BatchInputs( - seq_lens=torch.tensor([1, 1], dtype=torch.int32), - num_contexts=0, - request_ids=[0, 1], - prompt_lens=[3, 5], - ) - ) - assert cross.seq_lens.tolist() == [3, 1] - decoder.use_cross_sub_metadata(cross) - assert decoder.refresh_cross_query_side() is cross - assert cross.seq_lens is decoder.seq_lens and cross.num_contexts == 0 - assert cross.seq_lens_kv.tolist() == [7, 0] - - -class _Workspace(MetadataExtension): - fields = frozenset({"workspace_probe", "scratch_probe"}) - shared = frozenset({"workspace_probe"}) - - def __init__(self, metadata, *, source=None, **config) -> None: - super().__init__(metadata, source=source, **config) - self.workspace_probe = self.shared_or("workspace_probe", object) - self.scratch_probe = object() - - -def test_copies_are_constructed_and_take_shared_state_from_their_source(): - source = _metadata(_Workspace) - source.core().request_ids = [7] - copy_md = source.constructed_copy(max_num_tokens=8) - assert type(copy_md) is type(source) - assert copy_md.max_num_tokens == 8 and copy_md.request_ids == [7] - assert copy_md.core() is not source.core() - # Declared shared state is the source's; everything else is the copy's own. - assert copy_md.workspace_probe is source.workspace_probe - assert copy_md.scratch_probe is not source.scratch_probe - - -def _draft_batch(seq_lens, num_contexts) -> BatchInputs: - return BatchInputs( - seq_lens=torch.tensor(seq_lens, dtype=torch.int32), - num_contexts=num_contexts, - request_ids=list(range(len(seq_lens))), - prompt_lens=[5] * len(seq_lens), - ) - - -def test_draft_metadata_is_built_from_the_target_and_prepared_with_it(): - from tensorrt_llm._torch.attention.backends.features import DraftMetadata - - target = AttentionMetadata( - max_num_requests=2, max_num_tokens=4, one_model_draft=True, extension_types=(_Probe,) - ) - # Built at creation, after every other extension: the draft copies all of them. - assert type(target.extensions[-1]) is DraftMetadata - draft = target.draft - assert draft.find_ext(_Probe) is not None - target.prepare(_draft_batch([2, 1], 1)) - assert draft is target.draft and draft.seq_lens.tolist() == [2, 1] - assert draft.core() is not target.core() - # The draft has no draft of its own, and adopts nothing over a fresh family. - assert not draft.ext(DraftMetadata).one_model_draft - assert draft.find("draft_or_none", None) is None - assert draft.max_num_requests == 2 and draft.kv_cache_manager is target.kv_cache_manager - - # The target's prepare prepares the draft from the same batch. - target.prepare(_draft_batch([1, 1], 0)) - assert draft.seq_lens.tolist() == [1, 1] and draft.num_contexts == 0 - assert draft.seq_lens_cuda is not target.seq_lens_cuda - - # A draft loop changes only the draft's lengths and starts from the target's. - with target.draft_loop() as looped: - assert looped is draft - draft.set_seq_lens(slice(0, 2), device=3) - assert target.seq_lens_cuda.tolist() == [1, 1] - target.refresh_draft() - with target.draft_loop(): - assert draft.seq_lens_cuda.tolist() == [1, 1] - - -def test_metadata_without_one_model_draft_has_no_draft_metadata(): - target = AttentionMetadata(max_num_requests=1, max_num_tokens=1) - with pytest.raises(RuntimeError, match="no one-model draft"): - target.draft - - -def test_cuda_graph_copies_build_their_drafts_over_one_pool(): - target = AttentionMetadata(max_num_requests=2, max_num_tokens=4, one_model_draft=True) - target.prepare(_draft_batch([1, 1], 0)) - graphs = [target.create_cuda_graph_metadata(size, buffers=Buffers()) for size in (1, 2)] - drafts = [graph.draft for graph in graphs] - assert all(draft.is_cuda_graph for draft in drafts) - pools = {id(draft.cuda_graph_buffers) for draft in drafts} - assert len(pools) == 1 - assert drafts[0].cuda_graph_buffers is not graphs[0].cuda_graph_buffers - - -def _all_extension_classes(): - import tensorrt_llm._torch.attention.backends.sparse # noqa: F401 - import tensorrt_llm._torch.attention.backends.trtllm # noqa: F401 - - pending, found = [MetadataExtension], [] - while pending: - cls = pending.pop() - found.append(cls) - pending.extend(cls.__subclasses__()) - return found - - -def test_draft_lists_of_an_adopting_extension_do_not_overlap(): - overlapping = {} - for cls in _all_extension_classes(): - if not cls.adopts_target_state: - continue - bound = cls._declared_union("draft_bound") - written = cls._declared_union("draft_written") - private = cls._declared_union("draft_private") - common = (bound & written) | (bound & private) | (written & private) - if common: - overlapping[cls.__name__] = sorted(common) - assert overlapping == {} - - -def _production_extension_classes(): - import tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step # noqa: F401, E501 - import tensorrt_llm._torch.attention.backends.flashinfer # noqa: F401 - import tensorrt_llm._torch.attention.backends.fp4_mla.state # noqa: F401 - import tensorrt_llm._torch.attention.backends.sparse.csa2.metadata # noqa: F401 - import tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.metadata # noqa: F401 - import tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend # noqa: F401 - import tensorrt_llm._torch.attention.backends.sparse.minimax_m3.triton_metadata # noqa: F401 - import tensorrt_llm._torch.attention.backends.sparse.rocket.metadata # noqa: F401 - import tensorrt_llm._torch.attention.backends.vanilla # noqa: F401 - import tensorrt_llm._torch.models.modeling_qwen4_exp # noqa: F401 - - abstract = {MetadataExtension, MTPIndexShareUser, SparseAlgorithmExtension} - return { - cls.__name__: cls - for cls in _all_extension_classes() - if cls not in abstract and cls.__module__.startswith("tensorrt_llm.") - } - - -def test_resolve_extension_order_deduplicates_subclass_hierarchy(): - from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step import ( # noqa: E501 - GlmTp8DSAMetadata, - ) - from tensorrt_llm._torch.attention.backends.extensions import resolve_extension_order - - order = resolve_extension_order((DSAMetadata, HelixMetadata, GlmTp8DSAMetadata)) - assert DSAMetadata not in order - assert GlmTp8DSAMetadata in order - assert order.index(GlmTp8DSAMetadata) < order.index(HelixMetadata) - - class _DerivedProbe(_Probe): - pass - - metadata = _metadata(_Probe, _DerivedProbe) - assert type(metadata.ext(_Probe)) is _DerivedProbe - assert metadata.ext(_DerivedProbe) is metadata.ext(_Probe) - - class _SiblingA(HelixMetadata): - pass - - class _SiblingB(HelixMetadata): - pass - - with pytest.raises(TypeError, match="Ambiguous subclasses"): - resolve_extension_order((HelixMetadata, _SiblingA, _SiblingB)) - - -def test_every_extension_has_a_creation_config_check(): - """Each production extension names the test checking its constructed copies.""" - from utils.metadata_contract import CREATION_CONFIG_COVERAGE - - production = _production_extension_classes() - assert set(production) == set(CREATION_CONFIG_COVERAGE) - without_arguments = {name for name, test in CREATION_CONFIG_COVERAGE.items() if test is None} - assert all(production[name].declared_init_fields() == () for name in without_arguments) - - -# The hooks every extension implements: construction and the step's prepare. -_LIFECYCLE_HOOKS = frozenset({"__init__", "prepare"}) -# Members an extension may override: how it reports a creation argument, and -# the two draft members only DraftMetadata calls (adoption at the start of a -# draft loop, the refresh after the loop changed the draft's lengths). -_OPT_IN = frozenset({"creation_value", "adopt_target_state", "refresh_from_draft_lengths"}) -# Base methods no extension overrides: shared state and copy bookkeeping. -_BOOKKEEPING = frozenset( - { - "shared_or", - "_plan_draft_copies", - "creation_config", - } -) - - -def test_extensions_override_only_the_lifecycle_hooks(): - base_methods = { - name for name, value in vars(MetadataExtension).items() if inspect.isfunction(value) - } - assert base_methods - _BOOKKEEPING == _LIFECYCLE_HOOKS | _OPT_IN - forbidden = base_methods - _LIFECYCLE_HOOKS - _OPT_IN - overriding = {} - for name, cls in _production_extension_classes().items(): - overridden = forbidden & set(vars(cls)) - if overridden: - overriding[name] = sorted(overridden) - assert overriding == {} - - -def test_draft_member_overrides_call_their_base(): - for name, cls in _production_extension_classes().items(): - for member in ("adopt_target_state", "refresh_from_draft_lengths"): - override = vars(cls).get(member) - if override is not None: - assert f"super().{member}(" in inspect.getsource(override), (name, member) - - -# Ops and kernel launchers that write an attention metadata's length buffers, -# kept next to the ops: each of their arguments that is a length buffer must -# come from the core's lengths_for_kernel_update(), which marks the lengths -# changed so the next forward sees them refreshed. -_LENGTH_WRITING_OPS = frozenset( - { - "step0_epilogue", # speculative/fused_accept.py (Eagle3 / MTP-Eagle step 0) - "k3_spec_accept", # Kimi K3 fused acceptance (trtllm::k3_spec_accept) - "stage_steady_inputs", # SteadyMtpPrepare's staging kernel - "sample_step", # glm sm107 small-batch draft step - "draft_ehproj_table", # glm sm107 draft step on the decode kernels - "draft_sample_glue", - } -) -_LENGTH_BUFFERS = frozenset({"kv_lens_cuda", "seq_lens_cuda"}) -# Where the engine, models, modeling_v2 targets and speculative workers live. -_LENGTH_WRITER_ROOTS = ("pyexecutor", "models", "_experimental/modeling_v2", "speculative") - - -def _reads_length_buffer_directly(node) -> bool: - """Whether ``node`` is ``.kv_lens_cuda`` / ``.seq_lens_cuda``, sliced or not.""" - import ast - - while isinstance(node, ast.Subscript): - node = node.value - return isinstance(node, ast.Attribute) and node.attr in _LENGTH_BUFFERS - - -def test_kernels_write_length_buffers_only_from_lengths_for_kernel_update(): - import ast - import pathlib - - import tensorrt_llm._torch as torch_backend - - root = pathlib.Path(torch_backend.__file__).parent - offenders = [] - for base in _LENGTH_WRITER_ROOTS: - for path in (root / base).rglob("*.py"): - tree = ast.parse(path.read_text(encoding="utf-8")) - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - func = node.func - name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) - if name not in _LENGTH_WRITING_OPS: - continue - arguments = [*node.args, *(keyword.value for keyword in node.keywords)] - for argument in arguments: - candidates = [argument] - if isinstance(argument, ast.IfExp): - candidates = [argument.body, argument.orelse] - if any(_reads_length_buffer_directly(c) for c in candidates): - offenders.append(f"{path.relative_to(root)}:{node.lineno} {name}") - assert offenders == [] - - -@pytest.mark.parametrize("metadata_type", [TrtllmAttentionMetadata, VanillaAttentionMetadata]) -def test_context_rows_continue_as_generation_rows(metadata_type): - metadata = object.__new__(metadata_type) - core = metadata.core() - lengths = torch.tensor([5, 3, 4, 4], dtype=torch.int32) - core.bind_lengths(lengths, lengths.clone(), num_contexts=2) - if metadata_type is TrtllmAttentionMetadata: - # The per-row request types (0: context, 1: generation) and the - # cached-context-KV state, as a prepare of this batch leaves them. - core.host_request_types = torch.tensor([0, 0, 1, 1], dtype=torch.int32) - core.max_ctx_kv_len = 9 - core.num_ctx_cached_tokens = 4 - assert (metadata.num_contexts, metadata.num_ctx_tokens) == (2, 8) - - # The draft loop's step-0 update: one token per row from here on. - metadata.set_seq_lens(slice(0, 4), host=1, device=1) - metadata.as_generation_rows() - - assert metadata.num_contexts == 0 - assert metadata.num_generations == 4 - assert metadata.num_ctx_tokens == 0 - assert metadata.num_tokens == 4 - if metadata_type is TrtllmAttentionMetadata: - assert core.host_request_types.tolist() == [1, 1, 1, 1] - assert core.max_ctx_kv_len == 0 - assert core.num_ctx_cached_tokens == 0 - - -@needs_cuda -def test_prepare_applies_kv_lens_offsets_to_the_device_lengths_only(): - def batch(**changes): - # Attention without a KV cache: the KV lengths are the query lengths. - inputs = dict( - seq_lens=torch.tensor([6, 9], dtype=torch.int32), - num_contexts=2, - request_ids=[0, 1], - prompt_lens=[6, 9], - kv_cache_params=KVCacheParams(use_cache=False), - ) - return BatchInputs(**{**inputs, **changes}) - - metadata = TrtllmAttentionMetadata( - max_num_requests=2, max_num_tokens=8, max_seq_len=32, one_model_draft=True - ) - metadata.prepare(batch()) - buffer = metadata.kv_lens_cuda - - offsets = torch.tensor([-1, 2], dtype=torch.int32, device="cuda") - metadata.prepare(batch(kv_lens_offsets=offsets)) - assert metadata.kv_lens_cuda is buffer - assert metadata.kv_lens_cuda[:2].tolist() == [5, 11] - # The host lengths stay the scheduled ones; the device knows the correction. - assert metadata.kv_lens[:2].tolist() == [6, 9] - # The draft takes the corrected lengths when its loop starts. - start_draft_loop(metadata) - assert metadata.draft.kv_lens_cuda[:2].tolist() == [5, 11] - - # A step without corrections derives the lengths from the host alone. - metadata.prepare(batch()) - assert metadata.kv_lens_cuda[:2].tolist() == [6, 9] - - # A metadata without device KV lengths rejects the correction. - with pytest.raises(ValueError, match="kv_lens_offsets"): - _metadata().prepare(batch(kv_lens_offsets=offsets)) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py b/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py index 62df853eb95c..1aa7515ecffb 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_cache_manager.py @@ -471,33 +471,6 @@ def compute(request_ids, num_contexts): assert manager._swa_publication_token is manager._swa_publication is None -def test_unconsumed_swa_publication_does_not_reach_the_next_step(): - manager = CSA2CacheManager.__new__(CSA2CacheManager) - manager._swa_publication_token = manager._swa_publication = None - manager.pp_layers = [0] - manager.max_blocks_per_seq = 2 - manager.has_swa_cache = lambda layer: True - - def compute(request_ids, num_contexts): - manager._batch_page_tables = torch.zeros((1, len(request_ids), 2), dtype=torch.int32) - - manager.compute_batch_page_tables = compute - destination = torch.zeros((1, 2, 2, 2), dtype=torch.int32) - # A step publishes its SWA tables but no CSA2 prepare consumes them. - manager.prepare_batch_tables([0, 1], 0) - manager.copy_batch_block_offsets(destination, [0, 1], 1, 0, 2) - stale = manager.consume_swa_publication() - assert stale is not None - # The next step's preparation drops it before its own copy. - manager.prepare_batch_tables([0, 1], 0) - assert manager.consume_swa_publication() is None - assert manager._get_swa_publication(stale, destination, [0, 1], 0, 2) is None - # Its own copy publishes afresh. - manager.copy_batch_block_offsets(destination, [0, 1], 1, 0, 2) - fresh = manager.consume_swa_publication() - assert fresh is not None and fresh is not stale - - # Embedded DSpark linear verification replayed with the executor protocol. DRAFT_LEN = 5 diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py b/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py index ca1c957279a6..a1e50b7b55ee 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_metadata.py @@ -2403,24 +2403,3 @@ def forward(): _assert_swa_cpu_reference( metadata, manager, selected, starts, [1] * batch_size, [0] * batch_size ) - - -def test_bind_remote_tail_reads_context_requests(): - """The remote-tail role and starts come from the bound context requests.""" - - def request(mode, start): - return SimpleNamespace(py_csa2_remote_tail_mode=mode, py_csa2_remote_tail_start=start) - - csa2 = SimpleNamespace() - CSA2Metadata._bind_remote_tail(csa2, [request("destination", 64), request("destination", 8)]) - assert csa2.csa2_remote_tail_mode == "destination" - assert csa2.csa2_remote_tail_starts == (64, 8) - - CSA2Metadata._bind_remote_tail(csa2, [request(None, None)]) - assert csa2.csa2_remote_tail_mode is None - assert csa2.csa2_remote_tail_starts == () - - with pytest.raises(ValueError, match="homogeneous"): - CSA2Metadata._bind_remote_tail(csa2, [request("source", 1), request(None, None)]) - with pytest.raises(ValueError, match="homogeneous"): - CSA2Metadata._bind_remote_tail(csa2, [request("source", 1), request("destination", 2)]) diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py index c8951c7b9027..085c5e2d6067 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_token_major_rows.py @@ -283,55 +283,6 @@ def test_device_layout_preserves_mixed_context_row_and_token_prefixes() -> None: meta.metadata.check_lengths_current() -def test_device_layout_applies_kv_lens_delta_and_clears_lengths_changed(monkeypatch) -> None: - """Device-window reranking adjusts kv_lens_cuda, refreshes derived state, and leaves lengths clean.""" - torch.cuda.init() - meta = _metadata(num_pools=1, max_blocks=8, max_seqs=4) - block_offsets = torch.arange(1 * 4 * 2 * 8, dtype=torch.int32, device="cuda").reshape( - 1, 4, 2, 8 - ) - rows = _expand( - meta, - num_contexts=0, - row_req_idx=[0, 1, 1], - num_seqs=2, - block_offsets=block_offsets, - ) - assert rows == 3 - - md = meta.metadata - core = md.core() - core.num_generations = 2 - core._num_contexts = 0 - core._num_ctx_tokens = 0 - core._num_tokens = 3 - core.seq_lens = torch.tensor([2, 1], dtype=torch.int32) - core._seq_lens_cuda = torch.tensor([2, 1], dtype=torch.int32, device="cuda") - core.kv_lens = torch.tensor([104, 202], dtype=torch.int32) - core.kv_lens_cuda = torch.tensor([104, 202], dtype=torch.int32, device="cuda") - meta.index_share.req_idx_per_token = torch.full((32,), -1, dtype=torch.int32, device="cuda") - meta.indexer_k_cache_block_offsets = torch.arange( - 2 * 8, dtype=torch.int32, device="cuda" - ).reshape(2, 8) - refreshed = [] - monkeypatch.setattr(meta, "on_update_kv_lens", lambda: refreshed.append(True)) - - meta.apply_device_ragged_layout( - verify_lens=torch.tensor([1, 2], dtype=torch.int32, device="cuda"), - req_idx=torch.tensor([0, 1, 1], dtype=torch.long, device="cuda"), - kv_correction=torch.tensor([0, -1, 0], dtype=torch.int32, device="cuda"), - kv_lens_delta=torch.tensor([-2, 2], dtype=torch.int32, device="cuda"), - ) - - assert torch.equal(md.seq_lens_cuda[:2], torch.tensor([1, 2], dtype=torch.int32, device="cuda")) - assert torch.equal( - md.kv_lens_cuda[:2], torch.tensor([102, 204], dtype=torch.int32, device="cuda") - ) - assert refreshed == [True] - assert not md.lengths_changed - md.check_lengths_current() - - def test_every_runtime_view_shares_the_row_count() -> None: """All views the op reads must agree on batch_size, which is the row count. diff --git a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py index d4c576ff86d8..c2b78b4996e0 100644 --- a/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py +++ b/tests/unittest/_torch/attention/sparse/qsa/test_qsa_cache_manager_layout.py @@ -19,10 +19,7 @@ import pytest import torch -from utils.metadata_contract import check_copy_reproduces_creation_config -from tensorrt_llm._torch.attention.backends.sparse.qsa.metadata import QSAMetadata -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata from tensorrt_llm._torch.modules.qwen4_exp.cache_manager import ( QSA_INDEX_POSITION, QSA_SIDE_PLANES, @@ -321,23 +318,3 @@ def test_transfer_views_do_not_depend_on_the_kv_head_split(dtype: DataType) -> N assert roles == {str(Role.INDEX_KEY), str(QSA_INDEX_POSITION)} else: assert roles == {str(QSA_SIDE_PLANES)} - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") -def test_qsa_metadata_copies_reproduce_creation_config(): - """A constructed copy of a QSA metadata over the trunk manager reports its - creation configuration.""" - manager = _manager(DataType.BF16, draft=False) - try: - metadata = TrtllmAttentionMetadata( - max_num_requests=4, - max_num_tokens=8, - kv_cache_manager=manager, - extension_types=(QSAMetadata,), - sparse_metadata_params=QSASparseAttentionConfig().to_sparse_metadata_params( - pretrained_config=_CHECKPOINT - ), - ) - check_copy_reproduces_creation_config(metadata) - finally: - manager.shutdown() diff --git a/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py b/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py index 8480ebd9cade..0a822aa65dd9 100644 --- a/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py +++ b/tests/unittest/_torch/attention/sparse/rocketkv/test_rocketkv.py @@ -5,7 +5,6 @@ import pytest import torch from utils.llm_data import llm_models_root -from utils.metadata_contract import check_copy_reproduces_creation_config from utils.util import getSMVersion import tensorrt_llm @@ -13,7 +12,6 @@ from tensorrt_llm._torch.attention.backends.interface import AttentionForwardArgs from tensorrt_llm._torch.attention.backends.sparse.rocket import ( RocketKVCacheManager, - RocketMetadata, RocketTrtllmAttention, RocketTrtllmAttentionMetadata, RocketVanillaAttention, @@ -774,74 +772,6 @@ def test_rocket_add_dummy_requests_forwards_capture_sampling_params(mocker): assert base_add_dummy_requests.call_args.kwargs["capture_sampling_params"] is sampling_params -@pytest.mark.parametrize( - "metadata_cls", [RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata] -) -def test_rocket_copies_reproduce_creation_config(metadata_cls): - """A constructed copy of a prepared Rocket metadata reports its creation configuration.""" - sparse_attn_config = RocketSparseAttentionConfig( - window_size=32, kernel_size=3, prompt_budget=256, page_size=4, kt_cache_dtype="bfloat16" - ) - seq_lens, past_seen_tokens, request_ids = [300, 1], [0, 150], [0, 1] - kv_cache_manager = create_rocket_kv_cache_manager( - 1, 8, 128, 64, 4096, 2, DataType.BF16, sparse_attn_config - ) - kv_cache_manager.add_dummy_requests( - request_ids, [seq + past for seq, past in zip(seq_lens, past_seen_tokens)] - ) - try: - metadata = create_test_metadata( - seq_lens, - 1, - past_seen_tokens, - request_ids, - kv_cache_manager, - sparse_attn_config, - metadata_cls, - ) - assert metadata.find_ext(RocketMetadata) is not None - check_copy_reproduces_creation_config(metadata) - finally: - kv_cache_manager.shutdown() - - -@pytest.mark.parametrize( - "metadata_cls", [RocketTrtllmAttentionMetadata, RocketVanillaAttentionMetadata] -) -def test_rocket_prompt_budget_holds_across_repeated_prepares(metadata_cls): - """Preparing the same batch again (as CUDA graph warmup does) caps the cached - lengths at the prompt budget once, not once per prepare.""" - sparse_attn_config = RocketSparseAttentionConfig( - window_size=32, kernel_size=3, prompt_budget=256, page_size=4, kt_cache_dtype="bfloat16" - ) - seq_lens, past_seen_tokens, request_ids = [300, 1], [0, 400], [0, 1] - kv_cache_manager = create_rocket_kv_cache_manager( - 1, 8, 128, 64, 4096, 2, DataType.BF16, sparse_attn_config - ) - kv_cache_manager.add_dummy_requests( - request_ids, [seq + past for seq, past in zip(seq_lens, past_seen_tokens)] - ) - try: - metadata = create_test_metadata( - seq_lens, - 1, - past_seen_tokens, - request_ids, - kv_cache_manager, - sparse_attn_config, - metadata_cls, - ) - # The generation request's 401-token prompt is cut to the 256-token budget. - expected_cached = [0, 400 + 256 - 401] - for _ in range(3): - assert list(metadata.kv_cache_params.num_cached_tokens_per_seq) == expected_cached - if metadata_cls is RocketTrtllmAttentionMetadata: - assert metadata.kv_lens[:2].tolist() == [300, expected_cached[1] + 1] - metadata.prepare() - finally: - kv_cache_manager.shutdown() - - if __name__ == "__main__": # RocketKV e2e tests print("=== Testing RocketKV E2E tests ===") diff --git a/tests/unittest/_torch/attention/test_draft_metadata_reuse.py b/tests/unittest/_torch/attention/test_draft_metadata_reuse.py deleted file mode 100644 index 12bc4a5041d8..000000000000 --- a/tests/unittest/_torch/attention/test_draft_metadata_reuse.py +++ /dev/null @@ -1,780 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""A one-model draft metadata takes its target's prepared state. - -The draft (``md.draft``) of a target that shares its KV cache binds the -target's derived state when its loop starts and copies the tensors a draft -loop writes into its own buffers. For the dense TRTLLM backend and the DSA, -DeepSeek-V4 and CSA2 sparse families these tests check that the result equals -a draft that derives its own state in the target's prepare, that a draft step -leaves the target untouched, and that no draft tensor shares storage with a -buffer the target keeps apart from the draft. -""" - -import dataclasses -from types import SimpleNamespace - -import pytest -import torch -from _torch.helpers import refresh_from_lengths, start_draft_loop -from utils.metadata_contract import check_copy_reproduces_creation_config -from utils.util import getSMVersion - -from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig -from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures, BatchInputs -from tensorrt_llm._torch.metadata import KVCacheParams -from tensorrt_llm.bindings import DataType -from tensorrt_llm.bindings.internal.batch_manager import CacheType -from tensorrt_llm.mapping import Mapping - -pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") - -MAX_DRAFT_TOKENS = 3 -HISTORY = 300 - -# Bookkeeping of the adoption itself, present only on an adopting draft, and -# DSA's log-once marker of the fused decode-metadata kernel, set by whichever -# metadata ran that kernel first. -_ADOPTION_BOOKKEEPING = {"_draft_adopted_names", "_draft_copy_plan", "_fused_dsa_meta_armed"} - - -# --- Comparing metadata state ------------------------------------------------- - - -def _extensions_by_type(metadata): - return {type(extension): extension for extension in metadata._attached_extensions()} - - -def _tensor_items(name, value): - """``(key, tensor)`` of a tensor attribute or of each tensor in a dict attribute.""" - if isinstance(value, torch.Tensor): - yield name, value - elif isinstance(value, dict): - for key, item in value.items(): - if isinstance(item, torch.Tensor): - yield (name, key), item - - -def _extension_tensors(metadata): - """Every tensor of every extension, by (extension name, attribute key).""" - found = {} - for extension_type, extension in _extensions_by_type(metadata).items(): - for name, value in vars(extension).items(): - for key, tensor in _tensor_items(name, value): - found[(extension_type.__name__, key)] = tensor - return found - - -def _zero_buffers(metadata): - """Zero the uninitialized buffers so untouched regions compare equal.""" - for tensor in _extension_tensors(metadata).values(): - if tensor.is_floating_point() or tensor.dtype in ( - torch.int32, - torch.int64, - torch.int8, - torch.uint8, - torch.bool, - ): - tensor.zero_() - - -def _comparable(value): - if isinstance(value, torch.Tensor): - return ("tensor", value.shape, value.dtype, value.detach().cpu().clone()) - if value is None or isinstance(value, (bool, int, float, str)): - return value - if isinstance(value, (list, tuple, set, frozenset)) and all( - item is None or isinstance(item, (bool, int, float, str, tuple)) for item in value - ): - if isinstance(value, (set, frozenset)): - return tuple(sorted(value, key=repr)) - return tuple(value) - return None - - -def comparable_state(metadata): - """The comparable attributes of every extension; dictionaries entry by entry.""" - found = {} - for extension_type, extension in _extensions_by_type(metadata).items(): - for name, value in vars(extension).items(): - entries = ( - {(name, key): item for key, item in value.items()} - if isinstance(value, dict) - else {name: value} - ) - for key, item in entries.items(): - comparable = _comparable(item) - if comparable is not None: - found[(extension_type.__name__, key)] = comparable - return found - - -def _state_differences(actual, expected): - """The keys whose values differ or that only one of the states has.""" - different = [] - for key in sorted(set(actual) | set(expected), key=repr): - if key[1] in _ADOPTION_BOOKKEEPING: - continue - if key not in actual or key not in expected: - different.append(key) - continue - left, right = actual[key], expected[key] - if isinstance(left, tuple) and left and left[0] == "tensor": - if not ( - isinstance(right, tuple) - and right - and right[0] == "tensor" - and left[1:3] == right[1:3] - and torch.equal(left[3], right[3]) - ): - different.append(key) - elif left != right: - different.append(key) - return different - - -# --- Driving a target and its draft -------------------------------------------- - - -def _prepared(family, *, reuse, start_draft=True, batch=None, **creation_changes): - """A prepared one-model target, whose draft loop started with ``start_draft``. - - ``reuse=False`` makes its draft derive its own state in the target's - prepare; a draft taking the target's state takes it when its loop starts. - """ - metadata = family.backend.Metadata(**{**family.creation, **creation_changes}) - if not reuse: - metadata._draft_adopts = False - _zero_buffers(metadata) - _zero_buffers(metadata.draft) - metadata.prepare(batch or family.batch) - if start_draft: - start_draft_loop(metadata) - torch.cuda.synchronize() - return metadata - - -def _prepare_forward(family, metadata): - """What a forward builds before its first attention layer.""" - if family.num_heads is not None: - metadata.core().mla_prepare_scheduler_buffers(family.num_heads) - if family.forward is not None: - family.forward(metadata) - - -def _draft_step(family, draft, target): - """One MTP draft sub-step: one query token per request, one more KV token.""" - rows = slice(0, len(family.batch.request_ids)) - draft.core().copy_lengths_from(target.core()) - draft.core().set_seq_lens(rows, host=1, device=1, update_counts=True) - draft.core().adjust_kv_lens(rows, 1) - refresh_from_lengths(draft) - _prepare_forward(family, draft) - torch.cuda.synchronize() - - -# --- Families ------------------------------------------------------------------- - - -def _family(backend, creation, batch, **settings): - defaults = dict( - num_heads=None, - state=comparable_state, - banks=frozenset(), - forward=None, - extension_names=("TrtllmCoreMetadata",), - draft_manager=None, - ) - return SimpleNamespace( - backend=backend, creation=creation, batch=batch, **{**defaults, **settings} - ) - - -def _creation(manager, batch_size, **extra): - return dict( - max_num_requests=batch_size, - max_num_tokens=batch_size * (1 + MAX_DRAFT_TOKENS), - max_num_sequences=batch_size, - kv_cache_manager=manager, - mapping=Mapping(world_size=1, tp_size=1, rank=0), - runtime_features=AttentionRuntimeFeatures(has_speculative_draft_tokens=True), - spec_decoding=SpecDecodingConfig( - max_draft_len=MAX_DRAFT_TOKENS, max_total_draft_tokens=MAX_DRAFT_TOKENS - ), - one_model_draft=True, - **extra, - ) - - -def _batch(request_ids, cached): - """A generation batch; different cached lengths per request, so a mixed-up row shows.""" - return BatchInputs( - seq_lens=torch.full((len(request_ids),), 1 + MAX_DRAFT_TOKENS, dtype=torch.int), - num_contexts=0, - request_ids=list(request_ids), - prompt_lens=[HISTORY] * len(request_ids), - kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=cached), - ) - - -def _dense_manager(batch_size, leading_request_ids=()): - from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager - from tensorrt_llm.llmapi.llm_args import KvCacheConfig - - tokens_per_block = 32 - manager = KVCacheManager( - KvCacheConfig(max_tokens=(batch_size + 1) * 512, enable_block_reuse=False), - CacheType.SELF, - num_layers=1, - num_kv_heads=2, - head_dim=64, - tokens_per_block=tokens_per_block, - max_seq_len=512, - max_batch_size=batch_size + 1, - mapping=Mapping(world_size=1, tp_size=1, rank=0), - dtype=DataType.BF16, - ) - for ids in (list(leading_request_ids), list(range(batch_size))): - if ids: - manager.add_dummy_requests( - request_ids=ids, - token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * len(ids), - is_gen=False, - ) - return manager - - -@pytest.fixture(scope="module") -def dense(): - from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention - - batch_size = 4 - manager = _dense_manager(batch_size) - # A leading request moves the batch's blocks away from the target's. - draft_manager = _dense_manager(batch_size, leading_request_ids=[batch_size]) - yield _family( - TrtllmAttention, - _creation(manager, batch_size), - _batch(range(batch_size), [HISTORY - 7 * i for i in range(batch_size)]), - draft_manager=draft_manager, - ) - manager.shutdown() - draft_manager.shutdown() - - -@pytest.fixture(scope="module") -def dsa(): - from tensorrt_llm._torch.attention.backends.sparse.dsa import DSACacheManager - from tensorrt_llm._torch.attention.backends.utils import get_attention_backend - from tensorrt_llm._torch.model_config import ModelConfig - from tensorrt_llm.llmapi.llm_args import DeepSeekSparseAttentionConfig, KvCacheConfig - from tensorrt_llm.models.modeling_utils import QuantConfig - - batch_size, tokens_per_block = 8, 64 - mapping = Mapping(world_size=1, tp_size=1, rank=0) - sparse_config = DeepSeekSparseAttentionConfig( - index_n_heads=64, index_head_dim=128, index_topk=2048 - ) - pretrained_config = SimpleNamespace(rms_norm_eps=1e-6) - model_config = ModelConfig( - mapping=mapping, - sparse_attention_config=sparse_config, - pretrained_config=pretrained_config, - quant_config=QuantConfig(), - ) - model_config.extra_attrs["kv_cache_dtype"] = "auto" - manager = DSACacheManager( - KvCacheConfig( - max_tokens=batch_size * (HISTORY + 2 * tokens_per_block), - enable_block_reuse=False, - event_buffer_max_size=0, - ), - CacheType.SELFKONLY, - num_layers=2, - num_kv_heads=1, - head_dim=576, - tokens_per_block=tokens_per_block, - max_seq_len=1024, - max_batch_size=batch_size, - mapping=mapping, - dtype=DataType.BF16, - sparse_attention_config=sparse_config, - model_config=model_config, - vocab_size=129280, - ) - manager.add_dummy_requests( - request_ids=list(range(batch_size)), - token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * batch_size, - is_gen=False, - ) - backend = get_attention_backend( - "TRTLLM", sparse_params=sparse_config.to_sparse_params(pretrained_config=pretrained_config) - ) - creation = _creation( - manager, - batch_size, - enable_context_mla_with_cached_kv=False, - sparse_metadata_params=sparse_config.to_sparse_metadata_params( - pretrained_config=pretrained_config - ), - max_num_heads_per_kv=128, - ) - yield _family( - backend, - creation, - _batch(range(batch_size), [HISTORY - 7 * i for i in range(batch_size)]), - num_heads=128, - ) - manager.shutdown() - - -def _deepseek_v4_manager(batch_size, leading_request_ids=()): - from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import DeepseekV4CacheManager - from tensorrt_llm.llmapi.llm_args import KvCacheConfig, MTPDecodingConfig - - max_seq_len = 1024 - # The cache manager extends its config's compress ratios by the MTP layers. - sparse_config = _deepseek_v4_sparse_config() - manager = DeepseekV4CacheManager( - kv_cache_config=KvCacheConfig( - max_tokens=(batch_size + 1) * max_seq_len, - enable_block_reuse=False, - event_buffer_max_size=0, - ), - kv_cache_type=CacheType.SELFKONLY, - num_layers=len(sparse_config.compress_ratios), - num_kv_heads=1, - head_dim=512, - tokens_per_block=128, - max_seq_len=max_seq_len, - # Room for a draft cache's leading request. - max_batch_size=batch_size + 1, - max_input_len=max_seq_len, - mapping=Mapping(world_size=1, rank=0, tp_size=1, pp_size=1), - dtype=DataType.BF16, - compressor_dtype=DataType.FLOAT, - vocab_size=129280, - max_num_tokens=batch_size * max_seq_len, - sparse_attn_config=sparse_config, - spec_config=MTPDecodingConfig(max_draft_len=MAX_DRAFT_TOKENS), - ) - for ids in (list(leading_request_ids), list(range(batch_size))): - if ids: - manager.add_dummy_requests( - request_ids=ids, - token_nums=[HISTORY + 1 + MAX_DRAFT_TOKENS] * len(ids), - is_gen=False, - ) - return manager - - -def _deepseek_v4_sparse_config(): - from tensorrt_llm.llmapi.llm_args import DeepSeekV4SparseAttentionConfig - - return DeepSeekV4SparseAttentionConfig( - index_n_heads=64, - index_head_dim=128, - index_topk=512, - # Ratio-1, ratio-4 and ratio-128 layers, then the SWA-only MTP layer. - compress_ratios=[1, 4, 128, 1], - skip_indexer_for_short_seqs=False, - ) - - -@pytest.fixture(scope="module") -def deepseek_v4(): - from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4 import DeepseekV4TrtllmAttention - - batch_size = 4 - manager = _deepseek_v4_manager(batch_size) - draft_manager = _deepseek_v4_manager(batch_size, leading_request_ids=[batch_size]) - creation = _creation( - manager, - batch_size, - enable_context_mla_with_cached_kv=True, - sparse_metadata_params=_deepseek_v4_sparse_config().to_sparse_metadata_params(), - max_num_heads_per_kv=64, - ) - # The last request crosses a 128-token compression boundary. - cached = [HISTORY - 7 * i for i in range(batch_size - 1)] + [254] - yield _family( - DeepseekV4TrtllmAttention, - creation, - _batch(range(batch_size), cached), - num_heads=64, - extension_names=("TrtllmCoreMetadata", "DeepseekV4Metadata"), - draft_manager=draft_manager, - ) - manager.shutdown() - draft_manager.shutdown() - - -_CSA2_REQUEST_IDS = [41, 97] - - -def _csa2_manager(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.cache_manager import CSA2CacheManager - from tensorrt_llm._torch.attention.backends.sparse.csa2.params import CSA2Layout - from tensorrt_llm.llmapi.llm_args import DraftTargetDecodingConfig, KvCacheConfig - - manager = CSA2CacheManager( - KvCacheConfig( - enable_block_reuse=False, - max_tokens=4096, - enable_swa_scratch_reuse=True, - dtype="fp8", - ), - CacheType.SELFKONLY, - layout=CSA2Layout((0, 2, 2, 1), (1, 3), (1, 3)), - num_layers=4, - tokens_per_block=128, - mapping=Mapping(), - max_seq_len=512, - max_batch_size=len(_CSA2_REQUEST_IDS), - max_input_len=512, - max_num_tokens=1024, - dtype=DataType.BF16, - vocab_size=1024, - spec_config=DraftTargetDecodingConfig( - max_draft_len=MAX_DRAFT_TOKENS, speculative_model="attention-test-draft" - ), - ) - manager.add_dummy_requests( - request_ids=_CSA2_REQUEST_IDS, - token_nums=[HISTORY + 1, HISTORY - 6], - is_gen=True, - max_num_draft_tokens=MAX_DRAFT_TOKENS, - ) - manager._stream.synchronize() - return manager - - -# CSA2 buffer banks, host staging and device descriptor tables: their contents -# name addresses or hold scratch, so a draft's own differ from a derived one's. -_CSA2_ADDRESS_STATE = { - "_csa2_buffers", - "_csa2_host_staging", - "_csa2_owner_descriptors", - "_csa2_swa_descriptors", - "_csa2_upload_slab", -} - - -def _csa2_state(metadata): - """The comparable state, the compression batches' tensors included.""" - - found = { - key: value - for key, value in comparable_state(metadata).items() - if (key[1][0] if isinstance(key[1], tuple) else key[1]) not in _CSA2_ADDRESS_STATE - } - for owner, batch in getattr(metadata, "_csa2_compression", {}).items(): - for name in ( - "kv_page_table", - "score_page_table", - "kv_lengths", - "start_positions", - "cu_seq_lengths", - "cu_compressed_lengths", - ): - tensor = getattr(batch, name) - found[("CSA2Metadata", ("_csa2_compression", owner, name))] = ( - "tensor", - tensor.shape, - tensor.dtype, - tensor.cpu().clone(), - ) - found[("CSA2Metadata", ("_csa2_compression", owner, "sizes"))] = ( - batch.output_rows, - batch.page_size, - batch.max_compressed_per_request, - ) - return found - - -def _csa2_resolve_swa_slots(metadata): - """The SWA slots a forward's first CSA2 layer resolves.""" - metadata._ensure_swa_slots() - - -@pytest.fixture(scope="module") -def csa2(): - from tensorrt_llm._torch.attention.backends.sparse.csa2.backend import CSA2TrtllmAttention - - manager, draft_manager = _csa2_manager(), _csa2_manager() - yield _family( - CSA2TrtllmAttention, - _creation(manager, len(_CSA2_REQUEST_IDS)), - _batch(_CSA2_REQUEST_IDS, [HISTORY, HISTORY - 7]), - num_heads=64, - state=_csa2_state, - banks=frozenset({"_csa2_buffers"}), - forward=_csa2_resolve_swa_slots, - extension_names=("TrtllmCoreMetadata", "CSA2Metadata"), - draft_manager=draft_manager, - ) - manager.shutdown() - draft_manager.shutdown() - - -_BLACKWELL_ONLY = {"deepseek_v4", "csa2"} - - -@pytest.fixture(params=["dense", "dsa", "deepseek_v4", "csa2"]) -def family(request): - if request.param in _BLACKWELL_ONLY and getSMVersion() < 100: - pytest.skip(f"{request.param} needs Blackwell") - return request.getfixturevalue(request.param) - - -@pytest.fixture(params=["dense", "deepseek_v4", "csa2"]) -def family_with_draft_manager(request): - if request.param in _BLACKWELL_ONLY and getSMVersion() < 100: - pytest.skip(f"{request.param} needs Blackwell") - return request.getfixturevalue(request.param) - - -# --- Tests ---------------------------------------------------------------------- - - -def test_adopted_draft_equals_a_derived_draft(family): - adopted = _prepared(family, reuse=True) - derived = _prepared(family, reuse=False) - assert adopted.draft_adopts_target_state - assert not derived.draft_adopts_target_state - assert adopted.draft.kv_cache_manager is adopted.kv_cache_manager - - different = _state_differences(family.state(adopted.draft), family.state(derived.draft)) - assert different == [], different - _draft_step(family, adopted.draft, adopted) - _draft_step(family, derived.draft, derived) - different = _state_differences(family.state(adopted.draft), family.state(derived.draft)) - assert different == [], different - - -def test_loop_start_after_a_target_forward_equals_a_derived_draft(family): - """The step carries the overlap scheduler's KV-length correction - (``kv_lens_offsets``), which the target and a deriving draft apply in their - prepare and an adopting draft takes over at its loop's start.""" - batch_size = len(family.batch.request_ids) - # One draft token rejected for every request. - offsets = torch.full((batch_size,), -1, dtype=torch.int32, device="cuda") - batch = dataclasses.replace(family.batch, kv_lens_offsets=offsets) - drafts = [] - for reuse in (True, False): - metadata = _prepared(family, reuse=reuse, start_draft=False, batch=batch) - _prepare_forward(family, metadata) - draft = start_draft_loop(metadata) - assert draft is metadata.draft - torch.cuda.synchronize() - drafts.append(draft) - different = _state_differences(family.state(drafts[0]), family.state(drafts[1])) - assert different == [], different - for draft in drafts: - _prepare_forward(family, draft) - torch.cuda.synchronize() - different = _state_differences(family.state(drafts[0]), family.state(drafts[1])) - assert different == [], different - - -def test_draft_step_leaves_the_target_untouched(family): - metadata = _prepared(family, reuse=True) - before = family.state(metadata) - num_tokens = metadata.num_tokens - draft_kv_lens = metadata.draft.kv_lens_cuda.clone() - batch_size = len(family.batch.request_ids) - - _draft_step(family, metadata.draft, metadata) - - different = _state_differences(family.state(metadata), before) - assert different == [], different - assert metadata.num_tokens == num_tokens - assert metadata.draft.num_tokens == batch_size - assert torch.equal(metadata.draft.kv_lens_cuda[:batch_size], draft_kv_lens[:batch_size] + 1) - - -def test_written_buffers_keep_their_own_storage(family): - """No draft tensor shares storage with a written or private target tensor. - - A family's banks hold every buffer of their extension, the ones a draft - binds included, so they are not kept apart. - """ - metadata = _prepared(family, reuse=True) - target_extensions = _extensions_by_type(metadata) - kept_apart = set() - for extension_type, extension in _extensions_by_type(metadata.draft).items(): - source = target_extensions[extension_type] - written = extension_type._declared_union("draft_written") - private = extension_type._declared_union("draft_private") - for name in (written | private) - family.banks: - mine = dict(_tensor_items(name, vars(extension).get(name))) - for key, value in _tensor_items(name, vars(source).get(name)): - # The KV cache manager's own buffers are one object on both. - if mine.get(key) is not value: - kept_apart.add(value.untyped_storage().data_ptr()) - aliased = [ - key - for key, tensor in _extension_tensors(metadata.draft).items() - if tensor.untyped_storage().data_ptr() in kept_apart - ] - assert aliased == [], aliased - - -def test_every_attribute_of_an_adopting_extension_is_listed(family): - metadata = _prepared(family, reuse=True) - _draft_step(family, metadata.draft, metadata) - target_extensions = _extensions_by_type(metadata) - unlisted = {} - for extension_type, extension in _extensions_by_type(metadata.draft).items(): - assert extension_type.adopts_target_state, extension_type.__name__ - names = set(vars(extension)) | set(vars(target_extensions[extension_type])) - missing = extension_type.draft_unlisted(names) - if missing: - unlisted[extension_type.__name__] = sorted(missing) - assert unlisted == {}, unlisted - - -def test_draft_adoption_is_capture_safe(family): - """A draft loop's start takes the target's state again inside a CUDA graph - capture: only device copies, which a replay repeats.""" - metadata = _prepared(family, reuse=True) - expected = family.state(metadata.draft) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - start_draft_loop(metadata) - # The replay alone restores the device buffers a draft loop overwrote. - for extension_type, extension in _extensions_by_type(metadata.draft).items(): - for name in extension_type._declared_union("draft_written"): - for _, tensor in _tensor_items(name, vars(extension).get(name)): - if tensor.is_cuda: - tensor.zero_() - graph.replay() - torch.cuda.synchronize() - different = _state_differences(family.state(metadata.draft), expected) - assert different == [], different - - -def test_separate_draft_cache_reads_the_draft_manager(family_with_draft_manager): - """With a separate draft KV cache, the draft's state comes from the draft - manager: after the target's prepare and the draft loop's start, it equals a - metadata prepared over that manager directly.""" - from tensorrt_llm._torch.attention.backends.extensions import init_field_owner - from tensorrt_llm._torch.attention.backends.features.draft import _DRAFT_SETTINGS - - family = family_with_draft_manager - draft_manager = family.draft_manager - target = family.backend.Metadata(**family.creation, draft_kv_cache_manager=draft_manager) - draft_settings = { - name: value - for name, value in _DRAFT_SETTINGS.items() - if (owner := init_field_owner(name)) is not None and target.find_ext(owner) is not None - } - # A draft over its own cache stages the whole block table. - reference = family.backend.Metadata( - **{ - **family.creation, - "kv_cache_manager": draft_manager, - "one_model_draft": False, - "full_block_table_width": True, - **draft_settings, - } - ) - for metadata in (target, target.draft, reference): - _zero_buffers(metadata) - target.prepare(family.batch) - draft = start_draft_loop(target) - reference.prepare(family.batch) - torch.cuda.synchronize() - assert draft is target.draft - assert not target.draft_adopts_target_state - assert draft.kv_cache_manager is draft_manager - - def selected(metadata): - return { - key: value - for key, value in family.state(metadata).items() - if key[0] in family.extension_names - } - - different = _state_differences(selected(draft), selected(reference)) - assert different == [], different - - -def test_copies_reproduce_creation_config(family): - """Constructed copies of the target, its draft and its CUDA graph copy - report their sources' creation configuration, before and after prepare.""" - metadata = family.backend.Metadata(**family.creation) - check_copy_reproduces_creation_config(metadata) - metadata.prepare(family.batch) - torch.cuda.synchronize() - check_copy_reproduces_creation_config(metadata) - check_copy_reproduces_creation_config(metadata.draft) - graph = metadata.create_cuda_graph_metadata( - len(family.batch.request_ids), max_draft_tokens=MAX_DRAFT_TOKENS - ) - check_copy_reproduces_creation_config(graph) - - -def test_glm_tp8_dsa_draft_refresh_matches_base_dsa(dsa): - from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop.draft_step import ( # noqa: E501 - GlmTp8DSAMetadata, - step_ineligibility, - ) - from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata - - base = _prepared(dsa, reuse=True) - glm = _prepared(dsa, reuse=True, extension_types=(GlmTp8DSAMetadata,)) - assert type(glm.ext(DSAMetadata)) is GlmTp8DSAMetadata - assert type(glm.draft.ext(DSAMetadata)) is GlmTp8DSAMetadata - base_dsa = base.draft.ext(DSAMetadata) - glm_dsa = glm.draft.ext(DSAMetadata) - for extension in (base_dsa, glm_dsa): - extension.skip_indexer_for_gen_reqs = False - extension.skip_indexer_for_ctx_reqs = False - assert step_ineligibility(glm.draft, glm.draft.num_seqs, glm.draft.num_contexts) is None - - _draft_step(dsa, base.draft, base) - _draft_step(dsa, glm.draft, glm) - for name in ( - "slot_mapping_fp8", - "slot_mapping_scale", - "kv_lens_cuda_2d", - "scheduler_metadata_buffer", - "scheduler_metadata_buffer_full_next_n", - "gen_kv_indptr", - "gen_cached_token_indptr", - ): - torch.testing.assert_close(getattr(glm_dsa, name), getattr(base_dsa, name), rtol=0, atol=0) - torch.testing.assert_close( - glm_dsa.index_share.req_idx_per_token, - base_dsa.index_share.req_idx_per_token, - rtol=0, - atol=0, - ) - - -def test_csa2_draft_does_not_consume_an_swa_publication_of_its_own(csa2): - from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2Metadata - - if getSMVersion() < 100: - pytest.skip("csa2 needs Blackwell") - manager = csa2.creation["kv_cache_manager"] - consumed = [] - original = type(manager).consume_swa_publication - - def consume(self): - consumed.append(self) - return original(self) - - type(manager).consume_swa_publication = consume - try: - metadata = _prepared(csa2, reuse=True) - finally: - type(manager).consume_swa_publication = original - assert metadata.draft_adopts_target_state - # Only the target's CSA2 consumes the publication its core opened. - assert consumed == [manager] - assert manager.consume_swa_publication() is None - target, draft = metadata.ext(CSA2Metadata), metadata.draft.ext(CSA2Metadata) - assert draft.csa2_global_page_tables == target.csa2_global_page_tables - assert draft.csa2_positions is not target.csa2_positions diff --git a/tests/unittest/_torch/attention/test_metadata_creation_config.py b/tests/unittest/_torch/attention/test_metadata_creation_config.py deleted file mode 100644 index 08f68140531b..000000000000 --- a/tests/unittest/_torch/attention/test_metadata_creation_config.py +++ /dev/null @@ -1,176 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""A constructed copy of a metadata reports the creation configuration of its source. - -Covers the core families and the feature extensions that attach without a -family-specific KV cache manager; the sparse tests check their own metadata -(see ``utils.metadata_contract.CREATION_CONFIG_COVERAGE``). -""" - -import pytest -import torch -from utils.metadata_contract import check_copy_reproduces_creation_config - -import tensorrt_llm -from tensorrt_llm._torch.attention.backends.features import ( - FlashMlaMetadata, - HelixMetadata, - RecurrentConfig, - RecurrentMetadata, -) -from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig -from tensorrt_llm._torch.attention.backends.flashinfer import FlashInferAttentionMetadata -from tensorrt_llm._torch.attention.backends.interface import ( - AttentionMetadata, - AttentionRuntimeFeatures, - BatchInputs, -) -from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaMetadata, -) -from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.triton_metadata import ( - MiniMaxM3TritonMetadata, -) -from tensorrt_llm._torch.attention.backends.sparse.params import MTPIndexShareExtension -from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttentionMetadata -from tensorrt_llm._torch.attention.backends.vanilla import VanillaAttentionMetadata -from tensorrt_llm._torch.metadata import KVCacheParams -from tensorrt_llm._torch.models.modeling_qwen4_exp import Qwen4ExpPleState -from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager -from tensorrt_llm.bindings import DataType -from tensorrt_llm.llmapi.llm_args import KvCacheConfig -from tensorrt_llm.mapping import Mapping - -pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") - -BATCH_SIZE = 4 -TOKENS_PER_BLOCK = 32 -HISTORY = 100 -MAX_DRAFT_TOKENS = 1 - - -@pytest.fixture(scope="module") -def manager(): - manager = KVCacheManager( - KvCacheConfig(max_tokens=BATCH_SIZE * 8 * TOKENS_PER_BLOCK, enable_block_reuse=False), - tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, - num_layers=1, - num_kv_heads=2, - head_dim=64, - tokens_per_block=TOKENS_PER_BLOCK, - max_seq_len=512, - max_batch_size=BATCH_SIZE, - mapping=Mapping(world_size=1, tp_size=1, rank=0), - dtype=DataType.BF16, - ) - manager.add_dummy_requests( - request_ids=list(range(BATCH_SIZE)), token_nums=[HISTORY + 2] * BATCH_SIZE, is_gen=False - ) - yield manager - manager.shutdown() - - -def _batch(manager, query_len: int = 1 + MAX_DRAFT_TOKENS) -> BatchInputs: - return BatchInputs( - seq_lens=torch.full((BATCH_SIZE,), query_len, dtype=torch.int), - num_contexts=0, - request_ids=list(range(BATCH_SIZE)), - prompt_lens=[HISTORY] * BATCH_SIZE, - kv_cache_params=KVCacheParams( - use_cache=True, num_cached_tokens_per_seq=[HISTORY] * BATCH_SIZE - ), - ) - - -def test_trtllm_target_draft_and_graph_copies(manager): - metadata = TrtllmAttentionMetadata( - max_num_requests=BATCH_SIZE, - max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, - kv_cache_manager=manager, - runtime_features=AttentionRuntimeFeatures(has_speculative_draft_tokens=True), - spec_decoding=SpecDecodingConfig( - max_draft_len=MAX_DRAFT_TOKENS, max_total_draft_tokens=MAX_DRAFT_TOKENS - ), - one_model_draft=True, - ) - check_copy_reproduces_creation_config(metadata) - metadata.prepare(_batch(manager)) - check_copy_reproduces_creation_config(metadata) - check_copy_reproduces_creation_config(metadata.draft) - graph = metadata.create_cuda_graph_metadata(BATCH_SIZE, max_draft_tokens=MAX_DRAFT_TOKENS) - check_copy_reproduces_creation_config(graph) - graph.prepare(_batch(manager)) - check_copy_reproduces_creation_config(graph) - - -@pytest.mark.parametrize( - "extension_type", - [ - HelixMetadata, - FlashMlaMetadata, - MTPIndexShareExtension, - Qwen4ExpPleState, - MiniMaxM3MsaMetadata, - MiniMaxM3TritonMetadata, - ], - ids=lambda extension_type: extension_type.__name__, -) -def test_trtllm_feature_extensions(manager, extension_type): - metadata = TrtllmAttentionMetadata( - max_num_requests=BATCH_SIZE, - max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, - kv_cache_manager=manager, - extension_types=(extension_type,), - ) - check_copy_reproduces_creation_config(metadata) - metadata.prepare(_batch(manager, query_len=1)) - check_copy_reproduces_creation_config(metadata) - - -def test_vanilla_core(manager): - metadata = VanillaAttentionMetadata( - max_num_requests=BATCH_SIZE, - max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, - kv_cache_manager=manager, - ) - check_copy_reproduces_creation_config(metadata) - metadata.prepare(_batch(manager)) - check_copy_reproduces_creation_config(metadata) - - -def test_flashinfer_core(manager): - metadata = FlashInferAttentionMetadata( - max_num_requests=BATCH_SIZE, - max_num_tokens=(1 + MAX_DRAFT_TOKENS) * BATCH_SIZE, - kv_cache_manager=manager, - ) - check_copy_reproduces_creation_config(metadata) - metadata.prepare(_batch(manager)) - check_copy_reproduces_creation_config(metadata) - graph = metadata.create_cuda_graph_metadata(BATCH_SIZE) - check_copy_reproduces_creation_config(graph) - - -def test_recurrent_extension(): - metadata = AttentionMetadata( - max_num_requests=BATCH_SIZE, - max_num_tokens=BATCH_SIZE, - extension_types=(RecurrentMetadata,), - recurrent_config=RecurrentConfig(chunk_size=64), - ) - check_copy_reproduces_creation_config(metadata) - - -def test_cross_attention_decoder_and_sub_metadata(): - decoder = AttentionMetadata(max_num_requests=BATCH_SIZE, max_num_tokens=BATCH_SIZE) - decoder.prepare( - BatchInputs( - seq_lens=torch.tensor([3, 1], dtype=torch.int32), - num_contexts=1, - request_ids=[0, 1], - prompt_lens=[3, 5], - ) - ) - cross = decoder.create_cross_metadata([7, 0]) - check_copy_reproduces_creation_config(decoder) - check_copy_reproduces_creation_config(cross) diff --git a/tests/unittest/_torch/helpers.py b/tests/unittest/_torch/helpers.py index 1b4b76aace72..a2be8c838be4 100644 --- a/tests/unittest/_torch/helpers.py +++ b/tests/unittest/_torch/helpers.py @@ -426,14 +426,6 @@ def construct_extension(extension, **config): return extension -def start_draft_loop(metadata): - """Enter ``metadata.draft_loop()`` and return the draft, as a spec worker does. - - The loop stays open for the rest of the test (until the next start). - """ - return metadata.draft_loop().__enter__() - - def refresh_from_lengths(metadata): """Refresh ``metadata``'s length-derived state as a draft refresh does. diff --git a/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py b/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py index 766de1ab30b9..05281558dfe3 100644 --- a/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py +++ b/tests/unittest/_torch/pyexecutor/engine/test_decoder_runtime_hooks.py @@ -98,12 +98,6 @@ def test_a_default_msa_policy_arms_nothing(): assert metadata._model_metadata_config() == {} -def test_model_declared_extensions_and_settings_reach_creation(): - hooks = RuntimeHooks(metadata_config=(("pools", {}),)) - metadata = _metadata(hooks, object.__new__(TrtllmAttentionMetadata)) - assert metadata._model_metadata_config() == {"pools": {}} - - def test_an_msa_policy_requires_the_msa_backend_and_cache(): metadata = _metadata( RuntimeHooks(msa=MsaRuntimePolicy(reuse_pool_views=True)), diff --git a/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py b/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py index b910d143be76..bf30b136b9f6 100644 --- a/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py +++ b/tests/unittest/_torch/pyexecutor/test_kv_lens_update_once.py @@ -91,9 +91,3 @@ def test_uncorrected_steps_leave_the_inputs(spec, disable_overlap, with_kv_cache assert inputs["position_ids"].tolist() == [list(range(8))] assert metadata.kv_lens_cuda.tolist() == [100, 101] assert metadata.calls == [] - - -def test_the_retired_step_correction_hooks_are_gone() -> None: - hooks = RuntimeHooks() - assert not hasattr(hooks, "kv_lens_update_once") - assert not hasattr(hooks, "spec_offsets_op") diff --git a/tests/unittest/utils/metadata_contract.py b/tests/unittest/utils/metadata_contract.py deleted file mode 100644 index a0e743fc7700..000000000000 --- a/tests/unittest/utils/metadata_contract.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Shared checks of the attention metadata extension contracts.""" - -from typing import Any, Dict, Optional - -import torch - -# Every production metadata extension and the test module that checks its -# creation configuration with ``check_copy_reproduces_creation_config``; None -# for an extension that takes no creation arguments. -CREATION_CONFIG_COVERAGE: Dict[str, Optional[str]] = { - "AttentionCoreMetadata": "attention/test_metadata_creation_config.py", - "VanillaCoreMetadata": "attention/test_metadata_creation_config.py", - "TrtllmCoreMetadata": "attention/test_metadata_creation_config.py", - "DraftMetadata": "attention/test_metadata_creation_config.py", - "SpecDecodingMetadata": "attention/test_metadata_creation_config.py", - "HelixMetadata": "attention/test_metadata_creation_config.py", - "FlashMlaMetadata": "attention/test_metadata_creation_config.py", - "MTPIndexShareExtension": "attention/test_metadata_creation_config.py", - "Qwen4ExpPleState": "attention/test_metadata_creation_config.py", - "CrossAttentionMetadata": "attention/test_metadata_creation_config.py", - "RecurrentMetadata": "attention/test_metadata_creation_config.py", - "FlashInferCoreMetadata": "attention/test_metadata_creation_config.py", - "Fp4MlaState": None, - "DSAMetadata": "attention/test_draft_metadata_reuse.py", - "GlmTp8DSAMetadata": "attention/test_draft_metadata_reuse.py", - "DeepseekV4Metadata": "attention/test_draft_metadata_reuse.py", - "CSA2Metadata": "attention/test_draft_metadata_reuse.py", - "QSAMetadata": "attention/sparse/qsa/test_qsa_cache_manager_layout.py", - "RocketMetadata": "attention/sparse/rocketkv/test_rocketkv.py", - "MiniMaxM3MsaMetadata": "attention/test_metadata_creation_config.py", - "MiniMaxM3TritonMetadata": "attention/test_metadata_creation_config.py", -} - - -def _same(expected: Any, actual: Any) -> bool: - if isinstance(expected, torch.Tensor) or isinstance(actual, torch.Tensor): - if expected is actual: - return True - return ( - isinstance(expected, torch.Tensor) - and isinstance(actual, torch.Tensor) - and expected.shape == actual.shape - and expected.dtype == actual.dtype - and torch.equal(expected.cpu(), actual.cpu()) - ) - if expected is actual: - return True - try: - return bool(expected == actual) - except (RuntimeError, TypeError, ValueError): - return False - - -def check_copy_reproduces_creation_config(metadata) -> None: - """A copy built from ``metadata``'s configuration reports that configuration. - - For every attached extension, the copy's ``creation_config()`` equals the - source's: each creation argument survives ``constructed_copy`` as given. - """ - copy = metadata.constructed_copy() - differences = {} - for extension in metadata._attached_extensions(): - expected = extension.creation_config() - actual = copy.ext(type(extension)).creation_config() - names = sorted( - name - for name in expected.keys() | actual.keys() - if not _same(expected.get(name), actual.get(name)) - ) - if names: - differences[type(extension).__name__] = names - assert differences == {}, f"creation arguments a constructed copy changed: {differences}" From 770e555d22697ca749fb3d1772b9829b2837946c Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 07:54:53 -0800 Subject: [PATCH 4/7] [None][fix] Fix dynamic-tree slot reads, fused MTP spec flag and MSA tests Read the dynamic-tree slot storage's position_offsets and packed_mask under their own names, restore use_spec_decoding on the target metadata at the end of the fused GLM-5 MTP step as the stock worker does, and patch MSA predicates on the extension class in the MSA backend tests. Signed-off-by: Yihan Wang --- .../modeling.py | 1 + .../attention/backends/features/spec_decoding.py | 6 ++---- .../_torch/attention/sparse/msa/test_msa_backend.py | 11 +++++------ .../spec/test_modeling_v2_mtp_step.py | 1 + 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py index 15f81a293931..26ede56a821c 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/modeling.py @@ -2200,6 +2200,7 @@ def __call__( spec_metadata.batch_indices_cuda[:batch_size], num_accepted_tokens - 1 ] next_new_tokens = torch.cat([last_accepted.unsqueeze(1), next_draft_tokens], dim=1) + set_use_spec_decoding(attn_metadata, True) return { "logits": logits, "new_tokens": accepted_tokens, diff --git a/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py b/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py index ed9fb1f5022f..676858ab386c 100644 --- a/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py +++ b/tensorrt_llm/_torch/attention/backends/features/spec_decoding.py @@ -371,16 +371,14 @@ def update( if num_gens > 0: slot_ids = slot_storage.all_ids_buf[num_contexts:batch_size] - pos_src = torch.index_select( - slot_storage.spec_decoding_position_offsets, 0, slot_ids - )[:, :n_dt] + pos_src = torch.index_select(slot_storage.position_offsets, 0, slot_ids)[:, :n_dt] pos_dst = self.spec_decoding_position_offsets[: num_gens * n_dt].view( num_gens, n_dt ) pos_dst.copy_(pos_src, non_blocking=True) actual_mask_width = math.ceil(n_dt / 32) - mask_src = torch.index_select(slot_storage.spec_decoding_packed_mask, 0, slot_ids)[ + mask_src = torch.index_select(slot_storage.packed_mask, 0, slot_ids)[ :, :, :actual_mask_width ] if trtllm_gen: diff --git a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py index 6a3de8243fa8..73976346d05e 100644 --- a/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py +++ b/tests/unittest/_torch/attention/sparse/msa/test_msa_backend.py @@ -80,8 +80,8 @@ def test_msa_metadata_clears_padded_cache_slot_tail( metadata.msa_block_table = torch.zeros((1, 1), dtype=torch.int32) metadata.msa_seq_lens_cuda = torch.zeros(1, dtype=torch.int32) metadata.msa_subpage_block_table = None - metadata._msa_runs_no_fmha = lambda: True - metadata._msa_kv_lens_may_change = lambda: False + monkeypatch.setattr(MiniMaxM3MsaMetadata, "_msa_runs_no_fmha", lambda self: True) + monkeypatch.setattr(MiniMaxM3MsaMetadata, "_msa_kv_lens_may_change", lambda self: False) original_ptr = metadata.msa_out_cache_loc.data_ptr() for count in (4, 2, 0, 1): @@ -119,8 +119,7 @@ def test_msa_metadata_stages_query_rows_for_dynamic_kv_lens( from tensorrt_llm._torch.attention.backends.sparse.minimax_m3 import msa_backend monkeypatch.setattr(msa_backend, "maybe_pin_memory", lambda tensor: tensor) - metadata_cls = MiniMaxM3MsaSparseAttention.Metadata - metadata = metadata_cls.__new__(metadata_cls) + metadata = _msa_metadata() metadata._msa_buffers_ready = True # One four-token page per request, page id 10 + request id. metadata.kv_cache_manager = SimpleNamespace( @@ -138,8 +137,8 @@ def test_msa_metadata_stages_query_rows_for_dynamic_kv_lens( metadata.msa_q_intra = torch.full((max_tokens,), 99, dtype=torch.int32) metadata.msa_kv_lens_staged = torch.zeros(max_batch, dtype=torch.int32) metadata.msa_subpage_block_table = None - metadata._msa_runs_no_fmha = lambda: True - metadata._msa_kv_lens_may_change = lambda: True + monkeypatch.setattr(MiniMaxM3MsaMetadata, "_msa_runs_no_fmha", lambda self: True) + monkeypatch.setattr(MiniMaxM3MsaMetadata, "_msa_kv_lens_may_change", lambda self: True) # The second step is shorter, so a missed refresh would leave the first step's rows behind. for qo_lens in ([3, 0, 1, 2], [1, 2, 0]): diff --git a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py index d0aff5b23a33..cdf639ca2d7e 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py +++ b/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_nvfp4__sm_100__default_mtp_fused/spec/test_modeling_v2_mtp_step.py @@ -74,6 +74,7 @@ def __init__(self, ctx_lens, num_gens, draft_len, kv_lens, device): self.spec_decoding_position_offsets = None self.spec_decoding_generation_lengths = None self.use_spec_decoding = True + self.enable_flash_mla = False self.on_update() @property From 2e3adc19ce29234aa9871b3721d21ee13d4752f1 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 19:09:05 -0800 Subject: [PATCH 5/7] [None][feat] Fuse the GLM-5.2 draft-step length update into the model's DSA extension The tep4 and tp8 targets of GLM-5.2 sm_103 run their draft loop's per-step metadata update as one launch again. Each target's DSA extension provides run_draft_step(), which writes the draft's query and KV lengths through lengths_for_kernel_update(), the DSA decode state and the next positions in one kernel; the draft refresh after it only does the host side. Whether a loop is fused is decided once per forward; mixed batches and other forwards outside the domain keep the built-in loop. Also port the tep4 draft-loop test, add a tp8 draft-step test against the built-in update, and port the DSA offload prefetch test to build its metadata through the DSA extension. Signed-off-by: Yihan Wang --- .../spec_worker.py | 163 +++++++----- .../kernels/draft_loop/draft_loop.py | 22 +- .../kernels/draft_loop/draft_step.py | 204 +++++++-------- .../kernels/draft_loop/verify.py | 41 ++- .../dsa/test_dsa_offload_cache_manager.py | 65 +++-- .../test_modeling_v2_draft_loop.py | 57 +++-- .../glm_5_2_nvfp4__sm_103__tp8/__init__.py | 3 + .../test_modeling_v2_draft_step.py | 236 ++++++++++++++++++ 8 files changed, 542 insertions(+), 249 deletions(-) create mode 100644 tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/__init__.py create mode 100644 tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/test_modeling_v2_draft_step.py diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/spec_worker.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/spec_worker.py index 52e321cd6498..8885e34c0fdc 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/spec_worker.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/spec_worker.py @@ -18,12 +18,14 @@ * `accept`: the acceptance, the draft-0 input ids (the accepted tokens, flattened) and the step-0 gather rows; -* `step`: one metadata update after a draft step: the core's length edits, then the draft - refresh, whose DSA part `Glm52Tep4DSAMetadata` runs on `draft_step` (`MODE_REFRESH`); +* `step`: one metadata update after a draft step: `Glm52Tep4DSAMetadata.run_draft_step` + writes the draft's lengths, its DSA decode state and the next positions in one launch + (`draft_step`), then the draft refresh re-derives the rest; * `finalize`: `next_draft_tokens` and `next_new_tokens`. -The domain is checked on the host before any device work (`ineligibility`, -`step_ineligibility`); outside it the built-in code runs. `TRTLLM_GLM52_DRAFT_LOOP_VERIFY=1` +The domain is checked on the host once per forward, before any device work (`ineligibility` +for the forward, `draft_ineligibility` for the draft loop's metadata); outside it the built-in +code runs. `TRTLLM_GLM52_DRAFT_LOOP_VERIFY=1` runs the built-in code first on the live state, restores the device state, runs the fused op and counts every field that differs, on the device (`DraftLoopVerify`). """ @@ -53,8 +55,6 @@ # Annotation only: `modeling.py` imports this module. from .modeling import GlmMoeDsaTep4ModelingV2 -MODE_REFRESH = _k.MODE_REFRESH - @functools.lru_cache(maxsize=1) def enabled() -> bool: @@ -122,8 +122,9 @@ def ineligibility(worker, attn_metadata, spec_metadata) -> str | None: return None -def step_ineligibility(md, batch_size: int) -> str | None: - """Why the loop's draft refresh keeps the built-in DSA refresh, or None. Host only. +def draft_ineligibility(md, batch_size: int) -> str | None: + """Why this forward's draft loop keeps the built-in metadata updates, or None. Host only; + checked once, on the draft metadata, before the loop's first draft step. The domain is the GEN server's pure-decode DSA metadata: no context rows, the native fused decode metadata, uncompressed indexer lengths, no expanded MTP buffers, no ragged @@ -166,45 +167,74 @@ def step_ineligibility(md, batch_size: int) -> str | None: class Glm52Tep4DSAMetadata(DSAMetadata): - """The DSA metadata of this target: the draft loop's DSA refresh on `draft_step`. - - `refresh_from_draft_lengths` rebuilds the DSA decode state from the draft's lengths in - one launch (`MODE_REFRESH`) when the worker asked for it (`_fused_refresh`) and the step is - in the fused domain (`step_ineligibility`); otherwise it is the built-in refresh. - Declared in the target's `runtime_hooks`. + """The DSA metadata of this target, with the fused draft-step update of its draft loop. + + `run_draft_step` is this target's metadata update after a draft step: one `draft_step` + launch writes the draft's query and KV lengths (through the core's + `lengths_for_kernel_update`), the DSA decode state that follows them and the next step's + positions. The draft refresh after it (`refresh_from_draft_lengths`) then only does the + rest of the built-in DSA refresh: its host side, the GVR row reorder and the dense top-k + fallback. Any other refresh is the built-in one. `Glm52MTPEagleWorker` calls + `run_draft_step` on the forwards `draft_ineligibility` admits; declared in the target's + `runtime_hooks`. """ - fields = frozenset({"_fused_refresh"}) - draft_private = frozenset({"_fused_refresh"}) + fields = frozenset({"_draft_step_written"}) + draft_private = frozenset({"_draft_step_written"}) - _fused_refresh: bool = False + _draft_step_written: bool = False def __init__(self, metadata, *, source=None, **config) -> None: super().__init__(metadata, source=source, **config) - self._fused_refresh = False + self._draft_step_written = False - def refresh_from_draft_lengths(self) -> None: + def run_draft_step( + self, + step: int, + draft_len: int, + num_accepted: torch.Tensor, + position_ids: torch.Tensor | None, + gather_ids: torch.Tensor | None, + ) -> torch.Tensor | None: + """The update after draft step `step` of a pure-decode loop, before the draft refresh. + + Step 0 switches every row to one query token and takes the KV lengths back to the + accepted tokens (`kv += num_accepted - draft_len`); a later step advances them by one. + Returns `position_ids[gather_ids] + 1` (step 0) or `position_ids[:B] + 1`, or None + without `position_ids`.""" md = self.metadata - if not self._fused_refresh or step_ineligibility(md, md.num_seqs) is not None: + batch_size = md.num_seqs + if step == 0: + # The host side of the switch; the device lengths are the kernel's. + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + md.as_generation_rows() + md.lengths_for_kernel_update() + position_ids = _k.draft_step( + md, + mode=_k.MODE_STEP0 if step == 0 else _k.MODE_ADVANCE, + batch_size=batch_size, + draft_len=draft_len, + split_kv=_k.schedule_split_kv(self.num_sms, md.kv_lens_cuda.device), + num_accepted=num_accepted, + position_ids=position_ids, + gather_ids=gather_ids, + ) + self._draft_step_written = True + return position_ids + + def refresh_from_draft_lengths(self) -> None: + if not self._draft_step_written: super().refresh_from_draft_lengths() return - # The built-in refresh's host side; `draft_step` writes every device buffer its - # `on_update_kv_lens` writes (the indexer schedule included), then the GVR row - # reorder and the dense top-k fallback follow, as there. + self._draft_step_written = False + md = self.metadata + # The built-in refresh's host side; `run_draft_step` wrote every device buffer its + # `on_update_kv_lens` writes (the indexer schedule included). The GVR row reorder + # and the dense top-k fallback follow, as there. self._invalidate_length_prefixes() self.max_gen_seq_len = md.seq_lens[: md.num_seqs].max().item() self.indexer_schedule_stale = False self._invalidate_pool_view_cache() - _k.draft_step( - md, - mode=MODE_REFRESH, - batch_size=md.num_seqs, - draft_len=self.max_draft_tokens, - split_kv=_k.schedule_split_kv(self.num_sms, md.kv_lens_cuda.device), - num_accepted=None, - position_ids=None, - gather_ids=None, - ) self.gen_indexer_kv_lens_cuda_runtime = self.get_indexer_kv_lens( md.kv_lens_cuda[: md.num_seqs] ) @@ -216,29 +246,28 @@ def refresh_from_draft_lengths(self) -> None: def run_step( worker, md, step, batch_size, draft_len, num_accepted, position_ids, gather_ids, fused ): - """The metadata update after draft step `step` (the built-in loop's, for a batch without - context rows): the core's length edits and the draft refresh, its DSA part on `draft_step` - when `fused`. Returns the next step's positions, or None without `position_ids`.""" + """The metadata update after draft step `step` of a pure-decode loop, then the draft + refresh: on `Glm52Tep4DSAMetadata.run_draft_step` when `fused`, else the built-in loop's + (the core's length edits and the built-in refresh). Returns the next step's positions, or + None without `position_ids`.""" from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids - new_position_ids = None - if position_ids is not None: - rows = gather_ids if step == 0 else slice(0, batch_size) - new_position_ids = _select_mtp_position_ids(position_ids, rows) + 1 - if step == 0: - md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) - md.as_generation_rows() - md.adjust_kv_lens(slice(0, batch_size), num_accepted[:batch_size] - draft_len) + if fused: + new_position_ids = md.ext(Glm52Tep4DSAMetadata).run_draft_step( + step, draft_len, num_accepted, position_ids, gather_ids + ) else: - md.adjust_kv_lens(slice(0, batch_size), 1) - dsa = md.find_ext(Glm52Tep4DSAMetadata) - if dsa is not None: - dsa._fused_refresh = fused - try: - worker.refresh_draft() - finally: - if dsa is not None: - dsa._fused_refresh = False + new_position_ids = None + if position_ids is not None: + rows = gather_ids if step == 0 else slice(0, batch_size) + new_position_ids = _select_mtp_position_ids(position_ids, rows) + 1 + if step == 0: + md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) + md.as_generation_rows() + md.adjust_kv_lens(slice(0, batch_size), num_accepted[:batch_size] - draft_len) + else: + md.adjust_kv_lens(slice(0, batch_size), 1) + worker.refresh_draft() if step == 0: set_use_spec_decoding(md, False) return new_position_ids @@ -260,8 +289,6 @@ def __init__(self, worker, attn_metadata, spec_metadata) -> None: self.input_ids: torch.Tensor | None = None self.gather_ids: torch.Tensor | None = None self.next_new_tokens: torch.Tensor | None = None - # The metadata state is per forward: once a step declines, the rest stay built-in. - self.steps_engaged = True def accept(self, logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """The target's tokens (the worker's own pick), then the fused acceptance. @@ -314,20 +341,13 @@ def accept(self, logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: return self.accepted, self.num_accepted def step(self, md, step: int, position_ids) -> torch.Tensor | None: - """The metadata update after draft step `step`; the next step's positions.""" + """The fused metadata update after draft step `step`; the next step's positions.""" gather_ids = self.gather_ids if step == 0 else None - fused = self.steps_engaged - if fused: - reason = step_ineligibility(md, self.batch_size) - self._worker._glm_count("draft_step", reason is None) - if reason is not None: - log_decline("the metadata update", reason) - self.steps_engaged = fused = False args = (self._worker, md, step, self.batch_size, self.draft_len, self.num_accepted, position_ids, gather_ids) # fmt: skip - if fused and verify_enabled(): + if verify_enabled(): return VERIFY.step(*args) - return run_step(*args, fused=fused) + return run_step(*args, fused=True) def finalize(self, draft_tokens: list[torch.Tensor]) -> torch.Tensor | None: """`next_draft_tokens [B, K]`, with `next_new_tokens [B, K+1]` kept for the worker's @@ -648,13 +668,20 @@ def _forward_linear_draft_loop( accepted_tokens=None, pre_draft_seq_lens=None, ): # fmt: skip """The built-in MTP-Eagle draft loop restricted to the fused domain (no context rows, - no guided decoding, SA enhancer or LM-head TP: `ineligibility`), with the - metadata updates on `DraftLoop.step` and the stack on `DraftLoop.finalize`. + no guided decoding, SA enhancer or LM-head TP: `ineligibility`; the draft metadata: + `draft_ineligibility`), with the metadata updates on `DraftLoop.step` and the stack on + `DraftLoop.finalize`. Outside the domain, the built-in loop. Returns `(next_draft_tokens, next_new_tokens)` like the built-in loop; the fused path returns None for the second, and `_prepare_next_new_tokens` hands out the loop's.""" loop = self._glm_draft_loop - if loop is None or attn_metadata.find_ext(DSAMetadata) is None: + if loop is not None: + reason = draft_ineligibility(attn_metadata, batch_size) + self._glm_count("draft_step", reason is None) + if reason is not None: + log_decline("the metadata updates", reason) + loop = None + if loop is None: return super()._forward_linear_draft_loop( inputs, attn_metadata, spec_metadata, draft_model, draft_kv_cache_manager, num_contexts, batch_size, num_accepted_tokens, original_all_rank_num_tokens, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py index 35ebfa335259..48525e8a77fd 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_loop.py @@ -342,9 +342,19 @@ def step_update( ) -> Optional["_draft_step.DraftStepResult"]: """Fused per-step metadata update, or None for the native update. - ``refresh`` refreshes the draft after its lengths changed - (``md.refresh_draft()`` of the running draft loop). + Whether the loop's metadata updates are fused is decided once, at draft + step 0, from the draft metadata (``step_ineligibility``); the rest of + the loop follows. ``refresh`` refreshes the draft after its lengths + changed (``md.refresh_draft()`` of the running draft loop). """ + from ... import dispatch + + if step == 0: + reason = _draft_step.step_ineligibility(attn_metadata, self.batch_size, 0) + dispatch.log_capability( + "draft_loop", f"draft_step:{reason or 'engaged'}", reason is None, reason or "" + ) + self.step_engaged = reason is None if not self.step_engaged: return None @@ -361,13 +371,7 @@ def run_draft_loop(): refresh, ) - from ... import dispatch - - reason = _draft_step.step_ineligibility(attn_metadata, self.batch_size, 0) - dispatch.log_capability( - "draft_loop", f"draft_step:{reason or 'engaged'}", reason is None, reason or "" - ) - if _verify.verify_enabled() and reason is None: + if _verify.verify_enabled(): result = _verify.verified_step_update( run_draft_loop, refresh, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py index f30f47577887..75cdb2908618 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/draft_step.py @@ -23,11 +23,12 @@ and the fused decode metadata (request index, indexer K-cache slot mappings and generation indptrs). That is about twelve small launches per step. -``draft_step_update`` updates the core lengths through the core primitives and -``GlmTp8DSAMetadata.refresh_from_draft_lengths`` performs the DSA decode-state -refresh (plus the optional position-id advance) with one Triton launch. Every -device tensor it writes holds exactly the value the native chain writes. The -schedule is the DeepGEMM SM100 metadata partition +``draft_step_update`` performs the same update with +``GlmTp8DSAMetadata.run_draft_step``: one Triton launch writes the draft's query +and KV lengths (the core's ``lengths_for_kernel_update`` buffers), the DSA +decode state and the next step's position ids, and the draft refresh after it +does the rest of the native refresh on the host. Every device tensor it writes +holds exactly the value the native chain writes. The schedule is the DeepGEMM SM100 metadata partition (``deep_gemm/scheduler/sm100_paged_mqa_logits.cuh``): request work ``ceil(ctx / SPLIT_KV) * next_n``, SM ``i`` starts at work unit ``i * q + min(i, rem)`` and records ``(request_q_token_start, @@ -85,11 +86,12 @@ def _write_schedule( @triton.jit def _draft_step_kernel( - kv_lens_ptr, # int32 [>= B] (in) - seq_lens_ptr, # int32 [>= B] (in) + kv_lens_ptr, # int32 [>= B] (in/out) + seq_lens_ptr, # int32 [>= B] (out when STEP0, else in) + num_accepted_ptr, # int32 [B] (STEP0) pos_in_ptr, # int32/int64 position ids of this step (HAS_POS) pos_out_ptr, # same dtype [B] (out, HAS_POS) - gather_ptr, # int64 [B] rows of pos_in to advance (HAS_GATHER and HAS_POS) + gather_ptr, # int64 [B] rows of pos_in to advance (HAS_GATHER: STEP0 and HAS_POS) kv2d_ptr, # int32 [max_seqs, cap] (out) kv2d_stride, sched_ptr, # int32 [(NUM_SMS + 1) * 2] (out) @@ -105,6 +107,8 @@ def _draft_step_kernel( gen_cached_indptr_ptr, # int64 [>= B + 1] (out) num_tokens, B: tl.constexpr, + DRAFT_LEN: tl.constexpr, + STEP0: tl.constexpr, HAS_POS: tl.constexpr, HAS_GATHER: tl.constexpr, SKIP_SCHED: tl.constexpr, @@ -129,9 +133,22 @@ def _draft_step_kernel( offs_b = tl.arange(0, BLOCK_B) mask_b = offs_b < B - # ----- KV lengths and query lengths (already updated by core primitives) -- + # ----- KV lengths and query lengths (the loop's in-place updates) ------- kv = tl.load(kv_lens_ptr + offs_b, mask=mask_b, other=0) - seq = tl.load(seq_lens_ptr + offs_b, mask=mask_b, other=0) + # The lengths are updated in place, and each warp holds its own copy of them: without + # this barrier a warp that loads after another warp stored applies the update twice. + tl.debug_barrier() + if STEP0: + accepted = tl.load(num_accepted_ptr + offs_b, mask=mask_b, other=0) + kv = kv - (DRAFT_LEN - accepted) + seq = tl.full((BLOCK_B,), 1, tl.int32) + tl.store(seq_lens_ptr + offs_b, seq, mask=mask_b) + else: + kv = kv + 1 + seq = tl.load(seq_lens_ptr + offs_b, mask=mask_b, other=0) + tl.store(kv_lens_ptr + offs_b, kv, mask=mask_b) + kv = tl.where(mask_b, kv, 0) + seq = tl.where(mask_b, seq, 0) # ----- next step's position ids ------------------------------------------ if HAS_POS: @@ -296,82 +313,57 @@ class DraftStepResult: class GlmTp8DSAMetadata(DSAMetadata): - """DSA metadata extension for ``glm_5_2_nvfp4__sm_103__tp8`` draft-loop refresh. - - Replaces the multi-launch DSA draft-length refresh with one Triton launch - when the step falls inside the fused pure-decode domain, and delegates to - ``DSAMetadata.refresh_from_draft_lengths`` otherwise. + """DSA metadata extension for ``glm_5_2_nvfp4__sm_103__tp8`` draft loop. + + ``run_draft_step`` is this target's metadata update after a draft step: one + Triton launch writes the draft's query and KV lengths (through the core's + ``lengths_for_kernel_update``), the DSA decode state that follows them and + the next step's position ids. The draft refresh after it + (``refresh_from_draft_lengths``) then only does the host side of the native + DSA refresh. Any other refresh is ``DSAMetadata``'s. """ - fields = frozenset( - { - "_force_native_refresh", - "_pending_gather_ids", - "_pending_pos_in", - "_pending_pos_out", - } - ) + fields = frozenset({"_draft_step_written"}) adopts_target_state = True - draft_private = frozenset( - { - "_pending_pos_in", - "_pending_gather_ids", - "_pending_pos_out", - "_force_native_refresh", - } - ) + draft_private = frozenset({"_draft_step_written"}) - _pending_pos_in: Optional[torch.Tensor] = None - _pending_gather_ids: Optional[torch.Tensor] = None - _pending_pos_out: Optional[torch.Tensor] = None - _force_native_refresh: bool = False + _draft_step_written: bool = False def __init__(self, metadata, *, source=None, **config) -> None: super().__init__(metadata, source=source, **config) - self._pending_pos_in = None - self._pending_gather_ids = None - self._pending_pos_out = None - self._force_native_refresh = False + self._draft_step_written = False - def refresh_from_draft_lengths(self) -> None: - pos_in = self._pending_pos_in - gather_ids = self._pending_gather_ids - self._pending_pos_in = None - self._pending_gather_ids = None - self._pending_pos_out = None - md = self.metadata - if not self._force_native_refresh and ( - step_ineligibility(md, md.num_seqs, md.num_contexts) is None - ): - self._run_fused_refresh(pos_in, gather_ids) - return - if pos_in is not None: - from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids - - rows = gather_ids if gather_ids is not None else slice(0, md.num_seqs) - self._pending_pos_out = _select_mtp_position_ids(pos_in, rows) + 1 - super().refresh_from_draft_lengths() - - def _run_fused_refresh( - self, pos_in: Optional[torch.Tensor], gather_ids: Optional[torch.Tensor] - ) -> None: - md = self.metadata - batch_size = md.num_seqs - self._invalidate_length_prefixes() - if md.num_generations > 0: - self.max_gen_seq_len = md.seq_lens[md.num_contexts : md.num_seqs].max().item() - else: - self.max_gen_seq_len = 0 - skip_indexer_schedule = bool( + def _skips_indexer_schedule(self) -> bool: + # Every draft step after the first of an index-sharing draft loop reuses + # the TopK selection the first step captured (see DSAMetadata). + return bool( self.index_share.in_mtp_draft_loop and self.sparse_metadata_params.mtp_index_share ) - self.indexer_schedule_stale = skip_indexer_schedule - self._invalidate_pool_view_cache() - if not skip_indexer_schedule: - self.gen_indexer_kv_lens_cuda_runtime = md.kv_lens_cuda[:batch_size] - self.kv_lens_row_reorder = None + def run_draft_step( + self, + step: int, + draft_len: int, + num_accepted: torch.Tensor, + position_ids: Optional[torch.Tensor], + gather_ids: Optional[torch.Tensor], + ) -> Optional[torch.Tensor]: + """The update after draft step ``step`` of a pure-decode loop, before the refresh. + + Step 0 switches every row to one query token and takes the KV lengths + back to the accepted tokens (``kv += num_accepted - draft_len``); a later + step advances them by one. Returns ``position_ids[gather_ids] + 1`` + (step 0) or ``position_ids[:B] + 1``, or None without ``position_ids``. + """ + md = self.metadata + batch_size = md.num_seqs + if step == 0: + # The host side of the switch; the device lengths are the kernel's. + md.set_seq_lens(slice(0, batch_size), host=1, update_counts=True) + md.as_generation_rows() + lengths = md.lengths_for_kernel_update() + skip_indexer_schedule = self._skips_indexer_schedule() kv_mgr = md.kv_cache_manager index_head_dim = kv_mgr.index_head_dim use_fp4 = getattr(kv_mgr, "use_fp4", False) @@ -383,16 +375,17 @@ def _run_fused_refresh( expand = self.dsl_expand_factor if self.expand_for_dsl else 1 num_sms = self.num_sms block_offsets = self.indexer_k_cache_block_offsets - has_pos = pos_in is not None - has_gather = has_pos and gather_ids is not None - pos_out = torch.empty_like(pos_in[:batch_size]) if has_pos else None - self._pending_pos_out = pos_out + step0 = step == 0 + has_pos = position_ids is not None + has_gather = has_pos and step0 + pos_out = torch.empty_like(position_ids[:batch_size]) if has_pos else None dummy = md.kv_lens_cuda _draft_step_kernel[(1,)]( - md.kv_lens_cuda, - md.seq_lens_cuda, - pos_in if has_pos else dummy, + lengths["kv_lens_cuda"], + lengths["seq_lens_cuda"], + num_accepted if step0 else dummy, + position_ids if has_pos else dummy, pos_out if has_pos else dummy, gather_ids if has_gather else dummy, self.kv_lens_cuda_2d, @@ -410,6 +403,8 @@ def _run_fused_refresh( self.gen_cached_token_indptr, md.num_tokens, B=batch_size, + DRAFT_LEN=draft_len, + STEP0=step0, HAS_POS=has_pos, HAS_GATHER=has_gather, SKIP_SCHED=skip_indexer_schedule, @@ -432,6 +427,28 @@ def _run_fused_refresh( BLOCK_SM=triton.next_power_of_2(num_sms + 1), num_warps=_NUM_WARPS, ) + self._draft_step_written = True + return pos_out + + def refresh_from_draft_lengths(self) -> None: + if not self._draft_step_written: + super().refresh_from_draft_lengths() + return + self._draft_step_written = False + # The native refresh's host side; run_draft_step wrote every device + # buffer its on_update_kv_lens writes. + md = self.metadata + self._invalidate_length_prefixes() + if md.num_generations > 0: + self.max_gen_seq_len = md.seq_lens[md.num_contexts : md.num_seqs].max().item() + else: + self.max_gen_seq_len = 0 + skip_indexer_schedule = self._skips_indexer_schedule() + self.indexer_schedule_stale = skip_indexer_schedule + self._invalidate_pool_view_cache() + if not skip_indexer_schedule: + self.gen_indexer_kv_lens_cuda_runtime = md.kv_lens_cuda[: md.num_seqs] + self.kv_lens_row_reorder = None def step_ineligibility(md, batch_size: int, num_contexts: int) -> Optional[str]: @@ -517,12 +534,13 @@ def draft_step_update( position_ids: 1-D position ids of this step, or None on the last step. gather_ids: int64 ``[batch_size]`` rows of ``position_ids`` (step 0). + The caller checked the draft metadata once for the loop + (``step_ineligibility``). + Returns: ``DraftStepResult`` when the fused update ran, None when the caller must run the native update (no state has been modified then). """ - if step_ineligibility(md, batch_size, num_contexts) is not None: - return None if step == 0: if ( num_accepted_tokens is None @@ -537,24 +555,10 @@ def draft_step_update( if position_ids is not None and (position_ids.dim() != 1 or not position_ids.is_contiguous()): return None - if step == 0: - md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) - md.as_generation_rows() - md.adjust_kv_lens( - slice(num_contexts, batch_size), - num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, - ) - if num_contexts > 0: - md.adjust_kv_lens(slice(0, num_contexts), 1) - else: - md.adjust_kv_lens(slice(0, batch_size), 1) - - md._pending_pos_in = position_ids - md._pending_gather_ids = gather_ids if step == 0 else None - md._pending_pos_out = None + pos_out = md.ext(GlmTp8DSAMetadata).run_draft_step( + step, runtime_draft_len, num_accepted_tokens, position_ids, gather_ids + ) refresh() - pos_out = md._pending_pos_out - md._pending_pos_out = None if step == 0: md.use_spec_decoding = False return DraftStepResult(position_ids=pos_out) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py index eec6373614a7..a9e571a1c596 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/draft_loop/verify.py @@ -149,35 +149,30 @@ def _native_step_tail( position_ids, gather_ids, ): - """The native per-step tail of ``Eagle3OneModelWorker._forward_linear_draft_loop``.""" - from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids + """The native per-step tail of ``Eagle3OneModelWorker._forward_linear_draft_loop``. - from .draft_step import GlmTp8DSAMetadata + The draft refresh it ends with is ``DSAMetadata``'s: only ``run_draft_step`` + makes ``GlmTp8DSAMetadata`` skip it. + """ + from tensorrt_llm._torch.speculative.mtp import _select_mtp_position_ids new_position_ids = None if position_ids is not None: rows = gather_ids if i == 0 else slice(0, batch_size) new_position_ids = _select_mtp_position_ids(position_ids, rows) + 1 - dsa = md.find_ext(GlmTp8DSAMetadata) - if dsa is not None: - md._force_native_refresh = True - try: - if i == 0: - md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) - md.as_generation_rows() - md.adjust_kv_lens( - slice(num_contexts, batch_size), - num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, - ) - md.adjust_kv_lens(slice(0, num_contexts), 1) - refresh() - md.use_spec_decoding = False - else: - md.adjust_kv_lens(slice(0, batch_size), 1) - refresh() - finally: - if dsa is not None: - md._force_native_refresh = False + if i == 0: + md.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True) + md.as_generation_rows() + md.adjust_kv_lens( + slice(num_contexts, batch_size), + num_accepted_tokens[num_contexts:batch_size] - runtime_draft_len, + ) + md.adjust_kv_lens(slice(0, num_contexts), 1) + refresh() + md.use_spec_decoding = False + else: + md.adjust_kv_lens(slice(0, batch_size), 1) + refresh() return new_position_ids diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_offload_cache_manager.py b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_offload_cache_manager.py index ccd5c2908a10..3de8e28dd4df 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_offload_cache_manager.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_offload_cache_manager.py @@ -10,7 +10,7 @@ from tensorrt_llm._torch.attention.backends.sparse.dsa import DSACacheManagerV2 from tensorrt_llm._torch.attention.backends.sparse.dsa import custom_ops as dsa_custom_ops from tensorrt_llm._torch.attention.backends.sparse.dsa.backend import DSATrtllmAttention -from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAtrtllmAttentionMetadata +from tensorrt_llm._torch.attention.backends.sparse.dsa.metadata import DSAMetadata from tensorrt_llm._torch.attention.backends.sparse.dsa.module import forward_sparse_attn_custom_op from tensorrt_llm._torch.attention.backends.sparse.offload import ( SPARSE_OFFLOAD_STATE_COPIES, @@ -952,6 +952,26 @@ def forward(): manager.shutdown() +class _OffloadMetadata: + """The composed DSA metadata as the offload path reads it: the core's and the MTP + index-share names held here, every other name forwarded to the DSA extension.""" + + def __init__(self, dsa: DSAMetadata, **names) -> None: + self.__dict__.update(names, _dsa=dsa) + + def __getattr__(self, name): + return getattr(self.__dict__["_dsa"], name) + + def __setattr__(self, name, value) -> None: + if name in self.__dict__: + self.__dict__[name] = value + else: + setattr(self.__dict__["_dsa"], name, value) + + def find(self, name, default=None): + return self.__dict__.get(name, default) + + @pytest.mark.parametrize("dtype", [DataType.FP8, DataType.NVFP4]) @pytest.mark.parametrize("capture", [False, True], ids=["eager", "cuda-graph"]) @pytest.mark.parametrize("explicit_pattern", [False, True], ids=["glm52", "glm53"]) @@ -962,33 +982,34 @@ def test_shared_layer_prefetch_preserves_history(monkeypatch, dtype, capture, ex config.index_topk_pattern = ["F", "S", "S", "F", "S", "F", "S"] manager = _manager(dtype=dtype, pretrained_config=config) requests = [_request(17, 1), _request(18, 1)] - metadata = object.__new__(DSAtrtllmAttentionMetadata) - metadata.kv_cache_manager = manager - metadata.sparse_metadata_params = SimpleNamespace(enable_kv_cache_offload=True) - metadata.beam_width = 1 - metadata.max_draft_tokens = 0 - metadata.draft_kv_cache_manager = None - metadata.sparse_mla_topk = TOPK - metadata.is_cuda_graph = False - metadata.cuda_graph_buffers = {} - metadata.get_empty = lambda buffers, shape, dtype, **kwargs: torch.empty( - shape, dtype=dtype, device="cuda" + num_requests = len(requests) + dsa = object.__new__(DSAMetadata) + dsa.sparse_mla_topk = TOPK + dsa.max_draft_tokens = 0 + metadata = _OffloadMetadata( + dsa, + kv_cache_manager=manager, + sparse_metadata_params=SimpleNamespace(enable_kv_cache_offload=True), + beam_width=1, + is_cuda_graph=False, + cuda_graph_buffers={}, + get_empty=lambda buffers, shape, dtype, **kwargs: torch.empty( + shape, dtype=dtype, device="cuda" + ), + num_tokens=num_requests, + num_seqs=num_requests, + num_contexts=0, + shared_topk_indices=torch.full((num_requests, TOPK), -1, dtype=torch.int32, device="cuda"), + req_idx_per_token=torch.arange(num_requests, dtype=torch.int32, device="cuda"), + kv_lens_cuda=torch.empty(num_requests, dtype=torch.int32, device="cuda"), ) - metadata._init_dsa_sparse_offload_state() + dsa.metadata = metadata + dsa._init_dsa_sparse_offload_state() expected_pairs = {0: 1, 1: 2, 3: 4, 5: 6} if explicit_pattern else {2: 3, 3: 4, 4: 5} assert metadata.sparse_offload_next_shared_layer == expected_pairs state = metadata.sparse_offload_state # Inspect raw data and scale bytes separately; dequantization has its own test. state.nvfp4_scratch = None - num_requests = len(requests) - metadata._num_tokens = num_requests - metadata._seq_lens = torch.ones(num_requests, dtype=torch.int32) - metadata._num_contexts = 0 - metadata.shared_topk_indices = torch.full( - (num_requests, TOPK), -1, dtype=torch.int32, device="cuda" - ) - metadata.req_idx_per_token = torch.arange(num_requests, dtype=torch.int32, device="cuda") - metadata.kv_lens_cuda = torch.empty(num_requests, dtype=torch.int32, device="cuda") latent = torch.empty((num_requests, HEAD_DIM), dtype=torch.bfloat16, device="cuda") token_values = torch.empty((num_requests, 3), dtype=torch.int32, device="cuda") arenas = {} diff --git a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/test_modeling_v2_draft_loop.py b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/test_modeling_v2_draft_loop.py index 356621b899df..5818b47a3847 100644 --- a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/test_modeling_v2_draft_loop.py +++ b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/test_modeling_v2_draft_loop.py @@ -278,9 +278,13 @@ def as_generation_rows(self) -> None: def adjust_kv_lens(self, rows, delta) -> None: self.kv_lens_cuda[rows] += delta + def lengths_for_kernel_update(self) -> dict: + return {"seq_lens_cuda": self.seq_lens_cuda, "kv_lens_cuda": self.kv_lens_cuda} + class _RefreshWorker: - """The draft loop's `refresh_draft()`: the DSA extension's refresh of the draft.""" + """The draft loop's `refresh_draft()`: the DSA extension's refresh of the draft (the core + keeps no device state this metadata reads).""" def __init__(self, md: _Metadata) -> None: self._md = md @@ -319,7 +323,7 @@ def _metadata(batch, draft_len, expand, seed, seq_len, skip=False, gvr=False): ) # fmt: skip dsa = object.__new__(op.Glm52Tep4DSAMetadata) dsa.__dict__.update( - _fused_refresh=False, + _draft_step_written=False, _pool_cache_valid=True, _group_remap_batched={}, _ragged_mla_buffers_valid=True, @@ -341,6 +345,8 @@ def _metadata(batch, draft_len, expand, seed, seq_len, skip=False, gvr=False): use_expanded_buffers_for_mtp=False, expand_for_dsl=expand > 1, dsl_expand_factor=expand, + dsl_kernel_atoms=1, + dsl_kernel_next_n=0, kv_lens_expanded_cuda=torch.zeros((cap * 8,), dtype=torch.int32, device=DEV), indexer_k_cache_block_offsets=torch.randint( 0, 1 << 20, (cap, _MAX_BLOCKS), generator=g, dtype=torch.int32).to(DEV), @@ -459,8 +465,8 @@ def test_domain(monkeypatch) -> None: monkeypatch.setattr(dsa, "is_dsa_cache_manager", lambda m: m is not None) md = _metadata(4, 5, 1, 1, 6) - assert op.step_ineligibility(md, 4) is None - assert op.step_ineligibility(md, 3) is not None # not the batch the metadata holds + assert op.draft_ineligibility(md, 4) is None + assert op.draft_ineligibility(md, 3) is not None # not the batch the metadata holds for name, value in ( ("use_expanded_buffers_for_mtp", True), ("use_fp8_ds_mla", True), ("enable_ragged_verification", True), @@ -468,32 +474,29 @@ def test_domain(monkeypatch) -> None: ): # fmt: skip md = _metadata(4, 5, 1, 1, 6) setattr(md, name, value) - assert op.step_ineligibility(md, 4) is not None, name + assert op.draft_ineligibility(md, 4) is not None, name # The dense top-k fallback and the GVR row reorder stay in the domain. - assert op.step_ineligibility(_metadata(4, 5, 1, 1, 6, skip=True), 4) is None - assert op.step_ineligibility(_metadata(64, 5, 1, 1, 6, gvr=True), 64) is None + assert op.draft_ineligibility(_metadata(4, 5, 1, 1, 6, skip=True), 4) is None + assert op.draft_ineligibility(_metadata(64, 5, 1, 1, 6, gvr=True), 64) is None md = _metadata(4, 5, 1, 1, 6) md.num_contexts = 1 - assert op.step_ineligibility(md, 4) is not None - - -def test_step_declines_and_stays_declined(monkeypatch) -> None: - """Once a step declines, the loop's later steps keep the built-in update.""" - target, draft, batch_indices = _accept_case(4, 5, 3, torch.int32) - spec = SimpleNamespace( - draft_tokens=draft, batch_indices_cuda=batch_indices, is_cuda_graph=False, - is_ragged_verify=False, runtime_draft_len=5, - ) # fmt: skip - worker = _Worker(target, 0.0) - loop = op.DraftLoop(worker, SimpleNamespace(num_seqs=4), spec) - reasons = iter(["declined", None]) - monkeypatch.setattr(op, "step_ineligibility", lambda md, b: next(reasons)) - fused_flags = [] - monkeypatch.setattr(op, "run_step", lambda *args, fused: fused_flags.append(fused)) - loop.step(None, 0, None) - loop.step(None, 1, None) - assert fused_flags == [False, False] - assert worker.counts == {("draft_step", False): 1} + assert op.draft_ineligibility(md, 4) is not None + + +def test_declined_draft_metadata_runs_the_built_in_loop(monkeypatch) -> None: + """A forward whose draft metadata is outside the domain runs the built-in loop whole.""" + worker = object.__new__(op.Glm52MTPEagleWorker) + counts = {} + worker._glm_count = lambda route, hit: counts.update({(route, hit): 1}) + worker._glm_draft_loop = SimpleNamespace(step=None) # a fused acceptance ran + monkeypatch.setattr(op, "draft_ineligibility", lambda md, b: "declined") + built_in = object() + monkeypatch.setattr( + op.MTPEagleWorker, "_forward_linear_draft_loop", lambda self, *args: built_in + ) + args = ({}, None, None, None, None, 0, 4, None, None) + assert op.Glm52MTPEagleWorker._forward_linear_draft_loop(worker, *args) is built_in + assert counts == {("draft_step", False): 1} def test_worker_overrides_are_model_local() -> None: diff --git a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/__init__.py b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/__init__.py new file mode 100644 index 000000000000..81f065c4dace --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Package marker: test basenames repeat the other GLM-5.2 targets'.""" diff --git a/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/test_modeling_v2_draft_step.py b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/test_modeling_v2_draft_step.py new file mode 100644 index 000000000000..f305586a2ef6 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/test_modeling_v2_draft_step.py @@ -0,0 +1,236 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The tp8 target's fused draft-step update (`kernels/draft_loop/draft_step.py`) against the +engine's. + +A whole draft loop (step 0, then the advances) runs on two identical metadata objects: one +through the engine's per-step tail (the core's length edits and `DSAMetadata`'s draft refresh, +`verify._native_step_tail`), one through `draft_step_update` (`GlmTp8DSAMetadata.run_draft_step` +and the host-only refresh). Every device field, the host fields the next step reads and the +next positions must match exactly, with CUDA-graph padding rows (short KV lengths), the DSL +expansion on and off, and the index-share loop that skips the indexer schedule. +""" + +from types import SimpleNamespace + +import pytest +import torch + +if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3): + pytest.skip("the GLM-5.2 tp8 target imports sm_103 kernels", allow_module_level=True) + +from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop import ( # noqa: E402, E501 + draft_step as op, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.glm_moe_dsa.glm_5_2_nvfp4__sm_103__tp8.kernels.draft_loop import ( # noqa: E402, E501 + verify, +) +from tensorrt_llm._torch.attention.backends.sparse.params import ( # noqa: E402 + MTPIndexShareExtension, +) + +DEV = "cuda" +_TOKENS_PER_BLOCK = 64 +_MAX_BLOCKS = 4096 +_INDEX_HEAD_DIM = 128 +_TOPK = 2048 + + +class _Metadata: + """The composed metadata around one `GlmTp8DSAMetadata`: the core's lengths, counts and + length primitives, the MTP index-share tables, and every other name forwarded to the DSA + extension, as the composed metadata forwards it.""" + + def __init__(self, core: dict, index_share, dsa) -> None: + self.__dict__.update(core, _index_share=index_share, _dsa=dsa) + + @property + def num_seqs(self) -> int: + return self.seq_lens.shape[0] + + @property + def num_generations(self) -> int: + return self.num_seqs - self.num_contexts + + @property + def num_tokens(self) -> int: + return int(self.seq_lens.sum()) + + @property + def num_ctx_tokens(self) -> int: + return int(self.seq_lens[: self.num_contexts].sum()) + + def __getattr__(self, name): + owner = self.__dict__["_dsa"] + if name in type(self.__dict__["_index_share"]).declared_fields(): + owner = self.__dict__["_index_share"] + return getattr(owner, name) + + def __setattr__(self, name, value) -> None: + if name in self.__dict__: + self.__dict__[name] = value + else: + setattr(self.__dict__["_dsa"], name, value) + + def find_ext(self, extension_type): + for extension in (self._dsa, self._index_share): + if isinstance(extension, extension_type): + return extension + return None + + def ext(self, extension_type): + extension = self.find_ext(extension_type) + assert extension is not None, extension_type + return extension + + def set_seq_lens(self, rows, *, host=None, device=None, update_counts=False) -> None: + if host is not None: + self.seq_lens[rows] = host + if device is not None: + self.seq_lens_cuda[rows] = device + + def as_generation_rows(self) -> None: + self.num_contexts = 0 + + def adjust_kv_lens(self, rows, delta) -> None: + self.kv_lens_cuda[rows] += delta + + def lengths_for_kernel_update(self) -> dict: + return {"seq_lens_cuda": self.seq_lens_cuda, "kv_lens_cuda": self.kv_lens_cuda} + + +def _metadata(batch, draft_len, expand, seed, share): + """A composed metadata holding the GEN server's pure-decode buffers after the target's + verify step, built without a KV-cache manager (only the fields the update reads).""" + g = torch.Generator().manual_seed(seed) + cap = batch + 3 # the persistent buffers are sized for more requests than run + num_sms = torch.cuda.get_device_properties(0).multi_processor_count + rows = cap * (1 + draft_len) + kv = torch.randint(_TOPK + draft_len + 2, 200_000, (cap,), generator=g, dtype=torch.int32) + if batch > 2: # CUDA-graph padding rows: short KV lengths + kv[batch - 1] = draft_len + 1 + kv[batch - 2] = draft_len + 2 + core = dict( + seq_lens=torch.full((batch,), draft_len + 1, dtype=torch.int32), + seq_lens_cuda=torch.full((cap,), draft_len + 1, dtype=torch.int32, device=DEV), + kv_lens_cuda=kv.to(DEV), + num_contexts=0, + kv_cache_manager=SimpleNamespace( + index_head_dim=_INDEX_HEAD_DIM, use_fp4=False, quant_block_size=128 + ), + num_sparse_topk=_TOPK, + fp4_mla_state=None, + use_spec_decoding=True, + ) + index_share = object.__new__(MTPIndexShareExtension) + index_share.__dict__.update( + in_mtp_draft_loop=share, + req_idx_per_token=torch.full((rows,), -7, dtype=torch.int32, device=DEV), + ) + dsa = object.__new__(op.GlmTp8DSAMetadata) + dsa.__dict__.update( + _draft_step_written=False, + sparse_metadata_params=SimpleNamespace(mtp_index_share=share), + _pool_cache_valid=True, + _group_remap_batched={}, + _ragged_mla_buffers_valid=True, + nvfp4_mla_context_fp8_scratch=None, + use_fp8_ds_mla=False, + enable_ragged_verification=False, + ragged_verify_lens=None, + _indexer_compress_ratio=1, + runtime_tokens_per_gen_step=0, + max_draft_tokens=draft_len, + kv_lens_cuda_2d=torch.randint(0, 9, (cap, 1 + draft_len), generator=g, + dtype=torch.int32).to(DEV), + num_sms=num_sms, + scheduler_metadata_buffer=torch.zeros((num_sms + 1, 2), dtype=torch.int32, device=DEV), + scheduler_metadata_buffer_full_next_n=torch.zeros( + (num_sms + 1, 2), dtype=torch.int32, device=DEV), + scheduler_metadata_buffer_expanded=torch.zeros( + (num_sms + 1, 2), dtype=torch.int32, device=DEV), + use_expanded_buffers_for_mtp=False, + expand_for_dsl=expand > 1, + dsl_expand_factor=expand, + dsl_kernel_atoms=1, + dsl_kernel_next_n=0, + kv_lens_expanded_cuda=torch.zeros((cap * 8,), dtype=torch.int32, device=DEV), + indexer_k_cache_block_offsets=torch.randint( + 0, 1 << 20, (cap, _MAX_BLOCKS), generator=g, dtype=torch.int32).to(DEV), + slot_mapping_fp8=torch.full((rows,), -7, dtype=torch.int64, device=DEV), + slot_mapping_scale=torch.full((rows,), -7, dtype=torch.int64, device=DEV), + gen_kv_indptr=torch.full((cap + 1,), -7, dtype=torch.int64, device=DEV), + gen_cached_token_indptr=torch.full((cap + 1,), -7, dtype=torch.int64, device=DEV), + indexer_quant_block_size=128, + _tokens_per_block=_TOKENS_PER_BLOCK, + needs_gvr_prior=False, + kv_lens_row_reorder=None, + kv_lens_row_reorder_buffer=torch.full((cap,), -7, dtype=torch.int32, device=DEV), + skip_indexer_for_ctx_reqs=False, + skip_indexer_for_gen_reqs=False, + topk_indices_buffer=torch.full((rows, _TOPK), -7, dtype=torch.int32, device=DEV), + _fused_dsa_meta_armed=True, + indexer_schedule_stale=False, + max_gen_seq_len=draft_len + 1, + gen_indexer_kv_lens_cuda_runtime=None, + ) # fmt: skip + md = _Metadata(core, index_share, dsa) + dsa.metadata = md + index_share.metadata = md + return md + + +_HOST_FIELDS = ("num_contexts", "num_tokens", "num_generations", "use_spec_decoding", + "max_gen_seq_len", "indexer_schedule_stale", "kv_lens_row_reorder", + "_pool_cache_valid", "_ragged_mla_buffers_valid") # fmt: skip + + +def _device_fields(md) -> list[str]: + owners = (md.__dict__, vars(md._dsa), vars(md._index_share)) + return sorted( + n for owner in owners for n, v in owner.items() if isinstance(v, torch.Tensor) and v.is_cuda + ) + + +@pytest.mark.parametrize("share", [False, True]) +@pytest.mark.parametrize("expand", [1, 2]) +@pytest.mark.parametrize("draft_len", [1, 3, 5]) +@pytest.mark.parametrize("batch", [1, 2, 3, 8, 33, 64]) +def test_draft_step_update_matches_engine(batch, draft_len, expand, share, monkeypatch) -> None: + import tensorrt_llm._torch.attention.backends.sparse.dsa as dsa + + monkeypatch.setattr(dsa, "is_dsa_cache_manager", lambda manager: manager is not None) + seed = batch * 31 + draft_len * 7 + expand + 2 * share + native, fused = (_metadata(batch, draft_len, expand, seed, share) for _ in range(2)) + assert op.step_ineligibility(fused, batch, 0) is None + g = torch.Generator().manual_seed(seed) + num_accepted = torch.randint(1, draft_len + 2, (batch,), generator=g, dtype=torch.int32) + num_accepted = num_accepted.to(DEV) + gather_ids = torch.arange(batch, device=DEV) * (draft_len + 1) + num_accepted.long() - 1 + positions = torch.randint(0, 200_000, (batch * (draft_len + 1),), generator=g, + dtype=torch.int32).to(DEV) # fmt: skip + for step in range(draft_len): + pos = positions if step + 1 < draft_len else None + gather = gather_ids if step == 0 else None + want_pos = verify._native_step_tail( + native._dsa.refresh_from_draft_lengths, native, step, batch, 0, draft_len, + num_accepted, pos, gather, + ) # fmt: skip + result = op.draft_step_update( + fused, step, batch, 0, draft_len, num_accepted, pos, gather, + fused._dsa.refresh_from_draft_lengths, + ) # fmt: skip + assert result is not None + for name in _device_fields(native): + assert torch.equal(getattr(native, name), getattr(fused, name)), (step, name) + for name in _HOST_FIELDS: + want, got = getattr(native, name), getattr(fused, name) + if isinstance(want, torch.Tensor): + assert torch.equal(want, got), (step, name) + else: + assert want == got, (step, name) + if pos is not None: + assert torch.equal(want_pos, result.position_ids), step + positions = want_pos + else: + assert result.position_ids is None From 550a25847d1a89e512fe2b9aaaca381b7f26df93 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 19:23:59 -0800 Subject: [PATCH 6/7] [None][feat] Declare a model's metadata extensions as its metadata_extensions A model now declares the attention-metadata extensions it requires as its own metadata_extensions attribute, read by the engine through metadata_extensions_of(), instead of through RuntimeHooks. RuntimeHooks keeps only the extensions' creation arguments (metadata_config). The GLM-5.2 sm_103 tep4 and tp8 targets move to the attribute. Also restore the tep4 draft-loop kernel file to main's. Signed-off-by: Yihan Wang --- .../kernels/spec/draft_loop.py | 6 ++--- .../glm_5_2_nvfp4__sm_103__tep4/modeling.py | 6 ++--- .../glm_5_2_nvfp4__sm_103__tp8/modeling.py | 10 ++++++++ .../glm_5_2_nvfp4__sm_103__tp8/runtime.py | 6 ----- .../attention/ATTENTION_DEVELOPER_GUIDE.md | 7 +++--- .../_torch/pyexecutor/engine/metadata.py | 25 ++++++++++++++++--- .../engine/runners/decoder/metadata.py | 4 +-- .../_torch/pyexecutor/runtime_hooks.py | 6 ++--- 8 files changed, 46 insertions(+), 24 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/kernels/spec/draft_loop.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/kernels/spec/draft_loop.py index 9d2a36292375..c2a006dacf60 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/kernels/spec/draft_loop.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/kernels/spec/draft_loop.py @@ -396,9 +396,9 @@ def draft_step( ) -> torch.Tensor | None: """The device side of one metadata update of the draft loop, in one launch. - `md` is the loop's draft metadata in the fused domain (`spec_worker.step_ineligibility`); - every tensor written is the one the DSA extension's built-in draft refresh (with the loop's - length edits before it) writes. With + `md` is the loop's `DSAtrtllmAttentionMetadata` in the fused domain + (`spec_worker.step_ineligibility`); every tensor written is the one the built-in + `on_update_kv_lens` (with the loop's length edits before it) writes. With `position_ids`, also returns the next step's positions (`position_ids[gather_ids] + 1` in `MODE_STEP0`, `position_ids[:B] + 1` in `MODE_ADVANCE`).""" kv_mgr = md.kv_cache_manager diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/modeling.py index b9c5151890ef..d4720c731c42 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tep4/modeling.py @@ -56,7 +56,6 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_utils import DecoderModel, DecoderModelForCausalLM from tensorrt_llm._torch.modules.top_k import TopK, TopKImplementation -from tensorrt_llm._torch.pyexecutor.runtime_hooks import RuntimeHooks from tensorrt_llm._torch.speculative import ( MTPEagleWorker, get_spec_worker, @@ -5780,8 +5779,9 @@ def embed_tokens(self) -> torch.Tensor: class ModelingV2Glm52Nvfp4Sm103Tep4( DecoderModelForCausalLM[GlmMoeDsaTep4ModelingV2, PretrainedConfig] ): - # The DSA metadata with the draft loop's one-launch refresh (`spec_worker.py`). - runtime_hooks = RuntimeHooks(metadata_extensions=(Glm52Tep4DSAMetadata,)) + # The DSA metadata of this target, with the draft loop's one-launch `run_draft_step` + # (`spec_worker.py`). + metadata_extensions = (Glm52Tep4DSAMetadata,) def __init__(self, model_config: ModelConfig): cfg = model_config.pretrained_config diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/modeling.py index da93438bbc84..ad1d43a44250 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/modeling.py @@ -261,6 +261,16 @@ def __init__(self, model_config: ModelConfig): adopt_model(self, self.plan) dispatch.check_rank_uniform_adopted(adopted_state(self), world_size) + @property + def metadata_extensions(self) -> tuple: + """The attention-metadata extensions this target requires: with the plan's draft loop, + the DSA metadata with its one-launch draft-step update (``GlmTp8DSAMetadata``).""" + if not self.plan.enabled("draft_loop"): + return () + from .kernels.draft_loop.draft_step import GlmTp8DSAMetadata + + return (GlmTp8DSAMetadata,) + @property def runtime_hooks(self) -> RuntimeHooks: """The step-loop hooks this target declares, built on the engine's first read. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py index 1d907042f4c1..5c74eff17215 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/runtime.py @@ -74,14 +74,8 @@ def make_runtime_hooks(model: "GlmMoeDsaForCausalLM") -> RuntimeHooks: # The decode target takes the embedding rows of the prologue's launch from the # same object (the core's ``decode_prologue``). decode_prologue = model.model.decode_prologue = DecodePrologueHook() - metadata_extensions = () - if plan.enabled("draft_loop"): - from .kernels.draft_loop.draft_step import GlmTp8DSAMetadata - - metadata_extensions = (GlmTp8DSAMetadata,) return RuntimeHooks( loop_on_execution_stream=plan.enabled("execution_stream_loop"), decode_prologue=decode_prologue, executor_events=GlmExecutorEvents(plan), - metadata_extensions=metadata_extensions, ) diff --git a/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md index a8a257da1eea..49993377022f 100644 --- a/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/attention/ATTENTION_DEVELOPER_GUIDE.md @@ -393,9 +393,10 @@ opt-in and called only by `DraftMetadata` on a draft's extensions: refreshes the state derived from the draft's lengths after the draft loop changed them; an override calls `super()`. Length-derived state an extension computes lazily is invalidated by the extension itself in `prepare()` and -`refresh_from_draft_lengths()`. A model declares the extensions it needs and -their creation arguments through `RuntimeHooks.metadata_extensions` / -`metadata_config` (for example MiniMax-M3's `msa_policy`); its per-step state +`refresh_from_draft_lengths()`. A model declares the extensions it needs as +its `metadata_extensions` attribute (a subclass of an engine extension +replaces it), and their creation arguments through +`RuntimeHooks.metadata_config` (for example MiniMax-M3's `msa_policy`); its per-step state lives in a model extension or on the model, never in a module-level table keyed by the metadata. Extensions come from three places: diff --git a/tensorrt_llm/_torch/pyexecutor/engine/metadata.py b/tensorrt_llm/_torch/pyexecutor/engine/metadata.py index 190e6efdb3f6..cdaf13f556fb 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/metadata.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/metadata.py @@ -9,6 +9,7 @@ import torch +from tensorrt_llm._torch.attention.backends.extensions import MetadataExtension from tensorrt_llm._torch.attention.backends.features import FlashMlaMetadata, RecurrentConfig from tensorrt_llm._torch.attention.backends.features.spec_decoding import SpecDecodingConfig from tensorrt_llm._torch.attention.backends.interface import ( @@ -28,7 +29,7 @@ from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager -__all__ = ["build_attention_metadata"] +__all__ = ["build_attention_metadata", "metadata_extensions_of"] def _get_max_num_heads_per_kv(model_config: ModelConfig) -> int: @@ -50,6 +51,24 @@ def _get_max_num_heads_per_kv(model_config: ModelConfig) -> int: return 1 +def metadata_extensions_of(model: object) -> tuple[type[MetadataExtension], ...]: + """The attention-metadata extensions ``model`` requires, or (). + + A model declares them as ``metadata_extensions`` (a class or instance + attribute, or a property): a tuple of ``MetadataExtension`` types the + engine attaches to the model's attention metadata with its own, a + subclass of one of the engine's replacing it. + """ + extensions = tuple(getattr(model, "metadata_extensions", ())) + for extension_type in extensions: + if not (isinstance(extension_type, type) and issubclass(extension_type, MetadataExtension)): + raise TypeError( + f"{type(model).__name__}.metadata_extensions must hold MetadataExtension " + f"types, got {extension_type!r}" + ) + return extensions + + def build_attention_metadata( model_config: ModelConfig, *, @@ -77,8 +96,8 @@ def build_attention_metadata( ``max_seq_len`` bounds the sequences of no-KV-cache attention; it is given to the families whose core takes it. ``model_extension_types`` and ``model_extension_config`` are the extensions the model declares and their - creation arguments (``RuntimeHooks.metadata_extensions`` / - ``metadata_config``), attached with the engine's own. + creation arguments (``metadata_extensions_of(model)`` / + ``RuntimeHooks.metadata_config``), attached with the engine's own. """ pretrained_config = model_config.pretrained_config sparse_attention_config = model_config.sparse_attention_config diff --git a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py index 62e262aa1ee5..8055992c8031 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/runners/decoder/metadata.py @@ -30,7 +30,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping -from ...metadata import build_attention_metadata +from ...metadata import build_attention_metadata, metadata_extensions_of from .config import DecoderRunnerConfig if TYPE_CHECKING: @@ -121,7 +121,7 @@ def setup_attention( self._config.spec_config is not None and self._config.spec_config.spec_dec_mode.use_one_engine() ), - model_extension_types=self.runtime_hooks.metadata_extensions, + model_extension_types=metadata_extensions_of(self.model), model_extension_config=self._model_metadata_config(), ) self._arm_msa_policy(kv_cache_manager) diff --git a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py index a02a80715fb0..7bcfff8c93a5 100644 --- a/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py +++ b/tensorrt_llm/_torch/pyexecutor/runtime_hooks.py @@ -205,10 +205,8 @@ class RuntimeHooks: # With skip_unchanged_h2d: a changed packed 2-D upload stages only each row's changed column span (a page table that # gained a page re-sends a few bytes, not every padded row), so the pack keeps fitting the in-graph window. h2d_changed_span: bool = False - # Attention-metadata extensions the model's metadata carries, attached at creation (the engine's own choice of - # extensions is kept; a subclass of one replaces it), and their creation arguments, each routed by name to the - # extension that takes it, as (name, value) pairs. - metadata_extensions: Tuple[type, ...] = () + # Creation arguments of the model's attention-metadata extensions (its ``metadata_extensions`` and the engine's), + # each routed by name to the extension that takes it, as (name, value) pairs. metadata_config: Tuple[Tuple[str, Any], ...] = () # Run the overlap executor loop with the execution stream current, so the # loop's own stream work and the forward are ordered on one stream. From 1133923a5cfc9a64ee4b6dfbabe92c3eef6ba921 Mon Sep 17 00:00:00 2001 From: Yihan Wang Date: Wed, 11 Nov 2026 20:51:33 -0800 Subject: [PATCH 7/7] [None][chore] Fix stale metadata comments and restore a comment-only change Comments still described the removed draft swap and a before_forward() that does not exist; they now name the draft metadata and the calls that run on_update_kv_lens. The tp8 rope_kv_append comment is restored to main's wording, which still holds. Signed-off-by: Yihan Wang --- .../glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py | 2 +- .../minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py | 2 +- tensorrt_llm/_torch/attention/backends/trtllm.py | 2 +- .../attention/_m3_attn_runtime_tp4.py | 2 +- .../attention/test_modeling_v2_m3_draft_attention.py | 2 +- .../attention/_m3_attn_runtime_tp8.py | 2 +- .../attention/test_modeling_v2_m3_draft_attention.py | 2 +- .../attention/_m3_attn_runtime_tp2.py | 2 +- .../attention/test_modeling_v2_m3_draft_attention.py | 2 +- .../attention/test_modeling_v2_m3_n_valid_contract.py | 4 ++-- .../attention/_m3_attn_runtime_tp4.py | 2 +- .../attention/test_modeling_v2_m3_draft_attention.py | 2 +- .../attention/test_modeling_v2_m3_n_valid_contract.py | 4 ++-- .../attention/_m3_attn_runtime_tp8.py | 2 +- .../attention/test_modeling_v2_m3_draft_attention.py | 2 +- .../attention/test_modeling_v2_m3_n_valid_contract.py | 4 ++-- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py index cd01fb563af6..e6c1800b0f16 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_103__tp8/kernels/rope_kv_append/ops.py @@ -463,7 +463,7 @@ def remap_topk_to_global( Follows the native grouping rules: the leader of an active full+shared group computes the whole group in one launch and publishes it in - the DSA metadata's ``_group_remap_batched``; followers slice it; everything else + ``metadata._group_remap_batched``; followers slice it; everything else remaps per layer. """ from tensorrt_llm._torch.attention.backends.sparse.dsa import backend as dsa_backend diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py index 772b311a50b5..571aded15bb9 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/modeling.py @@ -4760,7 +4760,7 @@ def attention(self, x: torch.Tensor, md, reduce: bool) -> torch.Tensor: _record("m3_draft_attention", int(md.num_tokens)) if is_in_breakable_cuda_graph(): # A context step's draft pass is captured with the target's: the attention runs in an eager break, on the - # step's metadata (the drafter's: the runtime's draft swap installs it on the target's object). + # step's metadata (the drafter's: the draft loop makes it the model's attention_metadata). if get_model_extra_attrs()["attention_metadata"]() is not md: raise RuntimeError( "MiniMax-M3 drafter under a breakable prefill CUDA graph: its metadata is not the step's" diff --git a/tensorrt_llm/_torch/attention/backends/trtllm.py b/tensorrt_llm/_torch/attention/backends/trtllm.py index 5f21b9a08e4d..b2977388fd1c 100644 --- a/tensorrt_llm/_torch/attention/backends/trtllm.py +++ b/tensorrt_llm/_torch/attention/backends/trtllm.py @@ -113,7 +113,7 @@ class _ContextPlan: """Host state of a context-only trtllm-gen call, shared by the layers of one forward. TrtllmAttentionMetadata drops it whenever the lengths or the batch can change (prepare(), KV-length and - spec-dec updates, draft swaps); the bound tensors are also compared by identity. ``sig`` is the + spec-dec updates, draft refreshes); the bound tensors are also compared by identity. ``sig`` is the TrtllmAttention._context_plan_sig of the layer that built it. """ diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py index 28ad875449e9..c246cfb54dd9 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/_m3_attn_runtime_tp4.py @@ -4,7 +4,7 @@ One hybrid NVFP4 KV cache manager (``MiniMaxM3KVCacheManagerV2``) laid out as the target's: a dense layer in the FP8 half, a sparse MSA layer in the NVFP4 half with its FP8 index-K cache, and (``draft=True``) the one-model Eagle3 -drafter's shared layer, read through the manager's FP8 draft view as the runtime's draft swap reads it. Metadata is +drafter's shared layer, read through the manager's FP8 draft view as a draft metadata reads it. Metadata is the runtime's own type, filled by its own ``prepare()``; the backends are built the way the target builds them. The geometry is one rank's: 16 query heads over one KV head and one index head of dim 128. The mapping is a single diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/test_modeling_v2_m3_draft_attention.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/test_modeling_v2_m3_draft_attention.py index 222c0aa4fcef..0d4d913fae8a 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/test_modeling_v2_m3_draft_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp4/attention/test_modeling_v2_m3_draft_attention.py @@ -4,7 +4,7 @@ Every cell runs on the runtime's own objects (``_m3_attn_runtime_tp4``): the target's hybrid NVFP4 ``MiniMaxM3KVCacheManagerV2`` with its one-model Eagle3 draft layer, the requests allocated in it, drafter -``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager the runtime's draft swap installs) filled +``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager a draft metadata runs on) filled by its ``prepare()``, and the drafter's ``TrtllmAttention`` built as ``M3Eagle3Draft`` builds it (fused NeoX RoPE at the drafter checkpoint's theta, FP8 KV). Earlier rows go into the draft layer's cache through the in-tree scatter at the slots the manager assigned; the call rotates and appends the step's own. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py index a3c987b0453f..0e253f7c4f0a 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/_m3_attn_runtime_tp8.py @@ -4,7 +4,7 @@ One hybrid NVFP4 KV cache manager (``MiniMaxM3KVCacheManagerV2``) laid out as the target's: a dense layer in the FP8 half, a sparse MSA layer in the NVFP4 half with its FP8 index-K cache, and (``draft=True``) the one-model Eagle3 -drafter's shared layer, read through the manager's FP8 draft view as the runtime's draft swap reads it. Metadata is +drafter's shared layer, read through the manager's FP8 draft view as a draft metadata reads it. Metadata is the runtime's own type, filled by its own ``prepare()``; the backends are built the way the target builds them. The geometry is one rank's: 8 query heads over one KV head (M3's 4 KV heads, each duplicated on a rank pair) and diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/test_modeling_v2_m3_draft_attention.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/test_modeling_v2_m3_draft_attention.py index bfa051f8c35b..7bb98dc02aea 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/test_modeling_v2_m3_draft_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_103__tp8/attention/test_modeling_v2_m3_draft_attention.py @@ -4,7 +4,7 @@ Every cell runs on the runtime's own objects (``_m3_attn_runtime_tp8``): the target's hybrid NVFP4 ``MiniMaxM3KVCacheManagerV2`` with its one-model Eagle3 draft layer, the requests allocated in it, drafter -``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager the runtime's draft swap installs) filled +``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager a draft metadata runs on) filled by its ``prepare()``, and the drafter's ``TrtllmAttention`` built as ``M3Eagle3Draft`` builds it (fused NeoX RoPE at the drafter checkpoint's theta, FP8 KV). Earlier rows go into the draft layer's cache through the in-tree scatter at the slots the manager assigned; the call rotates and appends the step's own. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py index 78195491ca46..a36818f22cbd 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/_m3_attn_runtime_tp2.py @@ -4,7 +4,7 @@ One hybrid NVFP4 KV cache manager (``MiniMaxM3KVCacheManagerV2``) laid out as the target's: a dense layer in the FP8 half, a sparse MSA layer in the NVFP4 half with its FP8 index-K cache, and (``draft=True``) the one-model Eagle3 -drafter's shared layer, read through the manager's FP8 draft view as the runtime's draft swap reads it. Metadata is +drafter's shared layer, read through the manager's FP8 draft view as a draft metadata reads it. Metadata is the runtime's own type, filled by its own ``prepare()``; the backends are built the way the target builds them. The geometry is one rank's: 32 query heads over two KV heads and two index heads of dim 128 (one index-K head). diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_draft_attention.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_draft_attention.py index 29aa10b54689..cd0981a91c0b 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_draft_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_draft_attention.py @@ -4,7 +4,7 @@ Every cell runs on the runtime's own objects (``_m3_attn_runtime_tp2``): the target's hybrid NVFP4 ``MiniMaxM3KVCacheManagerV2`` with its one-model Eagle3 draft layer, the requests allocated in it, drafter -``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager the runtime's draft swap installs) filled +``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager a draft metadata runs on) filled by its ``prepare()``, and the drafter's ``TrtllmAttention`` built as ``M3Eagle3Draft`` builds it (fused NeoX RoPE at the drafter checkpoint's theta, FP8 KV). Earlier rows go into the draft layer's cache through the in-tree scatter at the slots the manager assigned; the call rotates and appends the step's own. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py index 342580bb875a..18d27f061f8d 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp2/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,8 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before - the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` (after the overlap scheduler's + correction) and its draft refresh call before the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py index 28ad875449e9..c246cfb54dd9 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/_m3_attn_runtime_tp4.py @@ -4,7 +4,7 @@ One hybrid NVFP4 KV cache manager (``MiniMaxM3KVCacheManagerV2``) laid out as the target's: a dense layer in the FP8 half, a sparse MSA layer in the NVFP4 half with its FP8 index-K cache, and (``draft=True``) the one-model Eagle3 -drafter's shared layer, read through the manager's FP8 draft view as the runtime's draft swap reads it. Metadata is +drafter's shared layer, read through the manager's FP8 draft view as a draft metadata reads it. Metadata is the runtime's own type, filled by its own ``prepare()``; the backends are built the way the target builds them. The geometry is one rank's: 16 query heads over one KV head and one index head of dim 128. The mapping is a single diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_draft_attention.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_draft_attention.py index a0be7f0dd6f4..ea45bab0c516 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_draft_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_draft_attention.py @@ -4,7 +4,7 @@ Every cell runs on the runtime's own objects (``_m3_attn_runtime_tp4``): the target's hybrid NVFP4 ``MiniMaxM3KVCacheManagerV2`` with its one-model Eagle3 draft layer, the requests allocated in it, drafter -``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager the runtime's draft swap installs) filled +``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager a draft metadata runs on) filled by its ``prepare()``, and the drafter's ``TrtllmAttention`` built as ``M3Eagle3Draft`` builds it (fused NeoX RoPE at the drafter checkpoint's theta, FP8 KV). Earlier rows go into the draft layer's cache through the in-tree scatter at the slots the manager assigned; the call rotates and appends the step's own. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py index 03df0439f559..8839ed8e1f36 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp4/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,8 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before - the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` (after the overlap scheduler's + correction) and its draft refresh call before the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py index a3c987b0453f..0e253f7c4f0a 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/_m3_attn_runtime_tp8.py @@ -4,7 +4,7 @@ One hybrid NVFP4 KV cache manager (``MiniMaxM3KVCacheManagerV2``) laid out as the target's: a dense layer in the FP8 half, a sparse MSA layer in the NVFP4 half with its FP8 index-K cache, and (``draft=True``) the one-model Eagle3 -drafter's shared layer, read through the manager's FP8 draft view as the runtime's draft swap reads it. Metadata is +drafter's shared layer, read through the manager's FP8 draft view as a draft metadata reads it. Metadata is the runtime's own type, filled by its own ``prepare()``; the backends are built the way the target builds them. The geometry is one rank's: 8 query heads over one KV head (M3's 4 KV heads, each duplicated on a rank pair) and diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_draft_attention.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_draft_attention.py index 7ee756142c77..924217374120 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_draft_attention.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_draft_attention.py @@ -4,7 +4,7 @@ Every cell runs on the runtime's own objects (``_m3_attn_runtime_tp8``): the target's hybrid NVFP4 ``MiniMaxM3KVCacheManagerV2`` with its one-model Eagle3 draft layer, the requests allocated in it, drafter -``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager the runtime's draft swap installs) filled +``TrtllmAttentionMetadata`` over the manager's FP8 draft view (the manager a draft metadata runs on) filled by its ``prepare()``, and the drafter's ``TrtllmAttention`` built as ``M3Eagle3Draft`` builds it (fused NeoX RoPE at the drafter checkpoint's theta, FP8 KV). Earlier rows go into the draft layer's cache through the in-tree scatter at the slots the manager assigned; the call rotates and appends the step's own. diff --git a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py index 342580bb875a..18d27f061f8d 100644 --- a/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py +++ b/tests/unittest/_torch/modeling_v2/models/minimax_m3/minimax_m3_nvfp4__sm_107__tp8/attention/test_modeling_v2_m3_n_valid_contract.py @@ -8,8 +8,8 @@ - the allocation (``_alloc_msa_proxy_scratch``); - the host-to-device copy in ``prepare()`` (``_build_step_plans``); -- ``on_update_kv_lens``, which the metadata's ``prepare()`` and ``before_forward()`` call before - the forward. It writes through the plain +- ``on_update_kv_lens``, which the metadata's ``prepare()`` (after the overlap scheduler's + correction) and its draft refresh call before the forward. It writes through the plain ``<<<>>>`` launch of ``msa_kv_update_kernel`` or through torch copies, so any PDL grid of the step starts only after it completes.