Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions scripts/generate_fmha_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@
"RopeParams": "AttentionRopeParams",
}

# Mirrored runtime classes whose members another class declares: the attention
# metadata keeps its state on its core and extensions, so a declaration-only
# class lists the members the native op reads, with their types.
MIRROR_DECLARATIONS = {
"TrtllmAttentionMetadata": "TrtllmAttentionMetadataNativeFields",
}

# Root fields whose C++ usage decides which mirrored members are emitted.
USAGE_PREFIXES = ("attn", "meta")

Expand Down Expand Up @@ -367,9 +374,10 @@ def _read_mirror(
struct_name = _struct_name(class_name)
if struct_name in mirrors:
return mirrors[struct_name]
if class_name not in classes:
raise ValueError(f"mirrored class {class_name} not found in the schema sources")
members = _collect_members(classes[class_name], classes)
declaring_class = MIRROR_DECLARATIONS.get(class_name, class_name)
if declaring_class not in classes:
raise ValueError(f"mirrored class {declaring_class} not found in the schema sources")
members = _collect_members(classes[declaring_class], classes)
if used is not None:
unknown = sorted({path[0] for path in used} - members.keys())
if unknown:
Expand Down
16 changes: 7 additions & 9 deletions tensorrt_llm/_torch/_experimental/modeling_v2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -326,15 +326,13 @@ mode the target uses.
reserves per sequence, and leave the rewind of rejected tokens to the
engine. Update `kv_lens` per draft step: step 0 drops the rejected drafts,
and each later step adds one.
* **Attention metadata.** Restore the attention metadata fields the stock
MTP-Eagle worker restores after its draft loop: the sequence lengths, the
speculative packed mask, position offsets and generation lengths, and
`all_rank_num_tokens`. CUDA graph warmup replays the forward on the same
metadata. Save the lengths with `prepare_for_spec_dec` and restore them with
`restore_from_spec_dec` in a `finally` block, so a failed step, such as an
out-of-memory error during warmup, does not leave them changed. Like the
worker, the draft chain leaves `num_contexts`, `host_request_types` and
`kv_lens_cuda` changed.
* **Attention metadata.** Run the draft chain on the draft metadata, inside
`with attn_metadata.draft_loop() as draft:`, as the stock MTP-Eagle worker

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Required] The stock MTP-Eagle worker does not do this: it uses self.draft_loop(attn_metadata) / self.refresh_draft() (eagle3.py:604, 1079; mtp.py:466, 990), the SpecWorkerBase wrappers that also install the draft as the model's attention_metadata extra attribute (speculative/interface.py:3592-3605). MLA, DSA and KDA custom ops resolve their metadata from that extra attribute (attention.py:47-65), so a target that follows this README with stock MLA/DSA/KDA attention runs its draft forwards on the target metadata. Please point targets at SpecWorkerBase.draft_loop / refresh_draft, and also document metadata_extensions and the lengths_for_kernel_update() rule for kernel-side length writes.

does. The draft starts from the target's state on every replay, including
CUDA graph warmup, and the target's metadata is not changed. Change the
draft's lengths through its core methods (`set_seq_lens`,
`as_generation_rows`, `adjust_kv_lens`) and call
`attn_metadata.refresh_draft()` before the next draft forward.
* **Per-token state.** Commit recurrent, conv or indexer state written per
token for the accepted tokens only, for example through the cache manager's
`update_mamba_states`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from torch import nn
from transformers import PretrainedConfig

from tensorrt_llm._torch.attention.backends.features.spec_decoding import set_use_spec_decoding
from tensorrt_llm._torch.attention.backends.interface import (
AttentionForwardArgs,
AttentionInputType,
Expand Down Expand Up @@ -2127,8 +2128,8 @@ class Glm5NextMtpStep:
3. Draft ``K = runtime_draft_len`` tokens: the MTP layer on (accepted token, target hidden state) rows, its
shared head on the vocabulary shard, and the global argmax across the TP group; step 0 runs on the verify
layout, steps 1.. on one row per request.
4. Restore the attention metadata the chain rewrote (CUDA-graph warmup replays this forward on the same
metadata) and return what the speculative sampler reads.
4. Return what the speculative sampler reads. The chain runs on the draft metadata (``draft_loop``), which
starts from the target's state on every replay, so the target's metadata is left as it was.

Every shape is fixed by host integers of the step (batch size, number of contexts, K), so the step captures in
one CUDA graph with the target.
Expand All @@ -2137,7 +2138,6 @@ class Glm5NextMtpStep:
def __init__(self, mapping: Mapping, vocab_size: int) -> None:
self.mapping = mapping
self.vocab_size = vocab_size
self._saved: dict[str, Any] = {}

def __call__(
self,
Expand Down Expand Up @@ -2183,29 +2183,24 @@ def __call__(
state_indices=attn_metadata.mamba_metadata.state_indices,
)

original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens
self._save_metadata(attn_metadata, batch_size)
try:
with attn_metadata.draft_loop() as draft_metadata:
next_draft_tokens = self._draft(
input_ids,
position_ids,
hidden_states,
accepted_tokens,
num_accepted_tokens,
attn_metadata,
draft_metadata,
spec_metadata,
draft_model,
)
finally:
self._restore_metadata(attn_metadata)
if original_all_rank_num_tokens is not None:
attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens

last_accepted = accepted_tokens[
spec_metadata.batch_indices_cuda[:batch_size], num_accepted_tokens - 1
]
next_new_tokens = torch.cat([last_accepted.unsqueeze(1), next_draft_tokens], dim=1)
attn_metadata.use_spec_decoding = True
set_use_spec_decoding(attn_metadata, True)
return {
"logits": logits,
"new_tokens": accepted_tokens,
Expand Down Expand Up @@ -2260,11 +2255,13 @@ def _draft(
hidden_states: torch.Tensor,
accepted_tokens: torch.Tensor,
num_accepted_tokens: torch.Tensor,
target_metadata: AttentionMetadata,
attn_metadata: AttentionMetadata,
spec_metadata: Any,
draft_model: Glm5NextDraftModel,
) -> torch.Tensor:
"""``next_draft_tokens [B, K]``. Rewrites the attention metadata's lengths as it goes; the caller restores."""
"""``next_draft_tokens [B, K]``. ``attn_metadata`` is ``target_metadata``'s draft metadata in its draft loop;
the chain advances its lengths as it goes."""
batch_size = attn_metadata.num_seqs
num_contexts = attn_metadata.num_contexts
num_gens = batch_size - num_contexts
Expand Down Expand Up @@ -2319,7 +2316,13 @@ def _draft(
step_positions[rows] if step_positions.dim() == 1 else step_positions[..., rows]
) + 1
self._advance_lengths(
attn_metadata, step, num_contexts, batch_size, draft_len, num_accepted_tokens
target_metadata,
attn_metadata,
step,
num_contexts,
batch_size,
draft_len,
num_accepted_tokens,
)
return torch.stack(drafts, dim=1)

Expand Down Expand Up @@ -2366,6 +2369,7 @@ def _draft_argmax(self, logits: torch.Tensor) -> torch.Tensor:

@staticmethod
def _advance_lengths(
target_metadata: AttentionMetadata,
attn_metadata: AttentionMetadata,
step: int,
num_contexts: int,
Expand All @@ -2376,18 +2380,14 @@ def _advance_lengths(
"""Lengths for the next draft step. After step 0 every request feeds one row: a generation request's KV
length drops its rejected drafts and a context request's grows by its first draft, then each step adds one."""
if step == 0:
attn_metadata._seq_lens[:batch_size].fill_(1)
attn_metadata._seq_lens_cuda[:batch_size].fill_(1)
attn_metadata.on_update()
attn_metadata.set_seq_lens(slice(0, batch_size), host=1, device=1, update_counts=True)
has_kv_cache = attn_metadata.kv_cache_manager is not None
if has_kv_cache and hasattr(attn_metadata, "host_request_types"):
attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1)
if has_kv_cache:
attn_metadata.num_contexts = 0
attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= (
draft_len - num_accepted_tokens[num_contexts:]
attn_metadata.as_generation_rows()
attn_metadata.adjust_kv_lens(
slice(num_contexts, batch_size), num_accepted_tokens[num_contexts:] - draft_len
)
attn_metadata.kv_lens_cuda[:num_contexts] += 1
attn_metadata.adjust_kv_lens(slice(0, num_contexts), 1)
if (
has_kv_cache
and num_contexts > 0
Expand All @@ -2396,42 +2396,11 @@ def _advance_lengths(
attn_metadata.block_ids_per_seq[:batch_size].copy_(
attn_metadata.kv_block_ids_per_seq[:batch_size], non_blocking=True
)
attn_metadata.update_for_spec_dec()
attn_metadata.use_spec_decoding = False
target_metadata.refresh_draft()
set_use_spec_decoding(attn_metadata, False)
else:
attn_metadata.kv_lens_cuda[:batch_size] += 1
attn_metadata.update_for_spec_dec()

# -- attention metadata save / restore ----------------------------------

def _save_metadata(self, attn_metadata: AttentionMetadata, batch_size: int) -> None:
"""Save what the draft chain overwrites: the sequence lengths (``prepare_for_spec_dec``) and the target's
speculative masks, position offsets and generation lengths."""
attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda")
saved = {}
if attn_metadata.spec_decoding_packed_mask is not None:
saved["packed_mask"] = attn_metadata.spec_decoding_packed_mask[:batch_size].clone()
if attn_metadata.spec_decoding_position_offsets is not None:
saved["position_offsets"] = attn_metadata.spec_decoding_position_offsets.clone()
if attn_metadata.spec_decoding_generation_lengths is not None:
saved["generation_lengths"] = attn_metadata.spec_decoding_generation_lengths[
:batch_size
].clone()
self._saved = saved

def _restore_metadata(self, attn_metadata: AttentionMetadata) -> None:
if attn_metadata.has_spec_dec_saved_state:
attn_metadata.restore_from_spec_dec()
attn_metadata.on_update()
saved, self._saved = self._saved, {}
if "packed_mask" in saved:
mask = saved["packed_mask"]
attn_metadata.spec_decoding_packed_mask[: mask.shape[0]].copy_(mask)
if "position_offsets" in saved:
attn_metadata.spec_decoding_position_offsets.copy_(saved["position_offsets"])
if "generation_lengths" in saved:
lengths = saved["generation_lengths"]
attn_metadata.spec_decoding_generation_lengths[: lengths.shape[0]].copy_(lengths)
attn_metadata.adjust_kv_lens(slice(0, batch_size), 1)
target_metadata.refresh_draft()


# ---------------------------------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@
from .ops.quantization.cute_dsl_act_quant_mxf8_rows import cute_dsl_act_quant_mxf8_rows
from .ops.quantization.cute_dsl_moe_quantize_nvfp4 import cute_dsl_moe_quantize_nvfp4
from .ops.sampling import cute_dsl_argmax as _argmax_op
from .spec_worker import Glm52MTPEagleWorker
from .spec_worker import Glm52MTPEagleWorker, Glm52Tep4DSAMetadata


def _build_step_args(md: TrtllmAttentionMetadata) -> dict:
Expand Down Expand Up @@ -5779,6 +5779,10 @@ def embed_tokens(self) -> torch.Tensor:
class ModelingV2Glm52Nvfp4Sm103Tep4(
DecoderModelForCausalLM[GlmMoeDsaTep4ModelingV2, PretrainedConfig]
):
# The DSA metadata of this target, with the draft loop's one-launch `run_draft_step`
# (`spec_worker.py`).
metadata_extensions = (Glm52Tep4DSAMetadata,)

def __init__(self, model_config: ModelConfig):
cfg = model_config.pretrained_config
super().__init__(
Expand Down
Loading
Loading