diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule.py index 457bb216f3f8..776ff3f2f6ef 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/ops/ssm/flashinfer_chunk_gated_delta_rule.py @@ -184,8 +184,9 @@ def _zero_fresh_states_kernel( def _chunk_inplace( - q, k, v, g, beta, scale, pool, indices, has_initial_state, cu_seqlens, output, linear_gate -) -> torch.Tensor: + q, k, v, g, beta, scale, pool, indices, has_initial_state, cu_seqlens, output, linear_gate, + all_carried=False, +) -> torch.Tensor: # fmt: skip """The short in-place indexed call: the fresh sequences' pool rows zeroed (one launch), then FlashInfer's chunked kernel (`use_cp=False`) reading and writing the named rows of the pool itself (`state_indices`, `output_state` the pool): no packed copy in or out.""" @@ -193,10 +194,13 @@ def _chunk_inplace( num_seqs = indices.shape[0] _, h, v_dim, k_dim = pool.shape - _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( - pool, indices, has_initial_state, *pool.stride(), - K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, - ) # fmt: skip + # `all_carried`: the caller knows on the host that every sequence carries a state, so + # the launch that zeroes the fresh ones would touch nothing. + if not all_carried: + _zero_fresh_states_kernel[(num_seqs, h, triton.cdiv(v_dim * k_dim, _GATHER_BLOCK))]( + pool, indices, has_initial_state, *pool.stride(), + K=k_dim, PLANE=v_dim * k_dim, BLOCK=_GATHER_BLOCK, num_warps=4, + ) # fmt: skip q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() out = ( output.squeeze(0) @@ -257,6 +261,77 @@ def _chunk_carried( return out.unsqueeze(0) +def flashinfer_chunk_gdn_inforward_split( + q, k, v, g, beta, pool, *, cu_seqlens, gather, has_initial_state, main_rows, live, + tail_tok, cu_tail, tail_slots, stage, stage_rows, stage_idx, output, + tail_lo=-1, tail_hi=-1, +) -> torch.Tensor: # fmt: skip + """A context-only chunk whose planned sequences end with an in-forward snapshot split + (`Mamba2Metadata.inforward_split`): what two context iterations compute, the chunk up to + the snapshot point P and then the tail `[P, prompt end)`, in one forward. + + `cu_seqlens` cuts every planned sequence at P, its tail a sequence of its own that starts + from zeros (`has_initial_state` false at its row of `gather`) and whose result is + discarded. The first pass is `_chunk_carried` over those sequences with the final states + of the real ones (`main_rows`) scattered to their slots `live`: the state at P. That + state is copied to the staging rows (`tail_slots`, int64, to `stage_rows`) and the tails + (`tail_tok` rows, `cu_tail`) are recomputed from it THERE by `_chunk_inplace` + (`stage_idx`: the staging rows in the pool's index dtype), the call a tail iteration of + its own makes, their outputs replacing the first pass's. Every tail carries a state, so + the launch that zeroes fresh ones is skipped. When exactly one sequence is split its + tail is the row slice `[tail_lo, tail_hi)`, read and written in place of five gathers + and a scatter; `(-1, -1)` takes the gathers. + + On return each planned slot holds the state at P, which the cache manager commits, and + `stage[stage_rows]` the state at the prompt end, which it copies back into the slot + after the commit (`finish_inforward_snapshots`). `g` is the linear gate. + """ + import flashinfer + + q3, k3, v3 = q.squeeze(0).contiguous(), k.squeeze(0).contiguous(), v.squeeze(0).contiguous() + g3, beta3 = g.squeeze(0), beta.squeeze(0).to(torch.float32) + packed = _gather_carried_states(pool, gather, has_initial_state) + out = output.squeeze(0) + final = torch.empty_like(packed) + flashinfer.chunk_gated_delta_rule( + q=q3, + k=k3, + v=v3, + g=g3, + beta=beta3, + scale=None, + initial_state=packed, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + output=out, + output_state=final, + ) + cast_scatter_fp32_vk_to_vk(final.index_select(0, main_rows), pool, live) + # The slots now hold the state at P and keep it: the tail advances the staged copy. + stage.index_copy_(0, stage_rows, pool.index_select(0, tail_slots)) + if tail_lo >= 0: + _chunk_inplace( + q3[tail_lo:tail_hi].unsqueeze(0), k3[tail_lo:tail_hi].unsqueeze(0), + v3[tail_lo:tail_hi].unsqueeze(0), g3[tail_lo:tail_hi].unsqueeze(0), + beta3[tail_lo:tail_hi].unsqueeze(0), + None, stage, stage_idx, None, cu_tail, out[tail_lo:tail_hi].unsqueeze(0), True, + all_carried=True, + ) # fmt: skip + else: + tail_out = _chunk_inplace( + q3.index_select(0, tail_tok).unsqueeze(0), + k3.index_select(0, tail_tok).unsqueeze(0), + v3.index_select(0, tail_tok).unsqueeze(0), + g3.index_select(0, tail_tok).unsqueeze(0), + beta3.index_select(0, tail_tok).unsqueeze(0), + None, stage, stage_idx, None, cu_tail, None, True, + all_carried=True, + ) # fmt: skip + out.index_copy_(0, tail_tok, tail_out.squeeze(0)) + return output + + class FlashinferChunkGatedDeltaRule(OpWrapper): """Run the gated delta rule over whole sequences, returning `(o, final_state)`. diff --git a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py index e6437bb637c1..4e1af472c546 100644 --- a/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py +++ b/tensorrt_llm/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/modeling.py @@ -398,6 +398,7 @@ ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.flashinfer_chunk_gated_delta_rule import ( flashinfer_chunk_gated_delta_rule, + flashinfer_chunk_gdn_inforward_split, ) from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.fused_recurrent_gated_delta_rule_cached_replay_update import ( # noqa: E501 fused_recurrent_gated_delta_rule_cached_replay_update, @@ -1724,6 +1725,14 @@ def _bind_cache_manager(self, md, position_ids) -> None: f"{rank_pools.ple_state[0].shape[0]} PLE slots (bind {self._binds})", flush=True, ) + # The tail of an in-forward snapshot split runs FlashInfer's in-place path. + max_tail = getattr(cm, "inforward_snapshot_max_tail", 0) + if getattr(cm, "inforward_snapshot_rows", 0) > 0 and max_tail > _GDN_INPLACE_MAX_TOKENS: + raise NotImplementedError( + f"in-forward Mamba snapshots with a tail of up to {max_tail} tokens " + "(kv_cache_config.mamba_state_config.additional_snapshot_offsets_from_end): this " + f"target splits tails of at most {_GDN_INPLACE_MAX_TOKENS} tokens" + ) bound = _Bound(cm, rank_pools) if self.spec_rollback is not None: self.spec_rollback.set_index_planes( @@ -3754,6 +3763,110 @@ def observe(self, layer: int, k: torch.Tensor, v: torch.Tensor) -> bool: # ------------------------------------------------------------------ prefill --- +# TRTLLM_QWEN4_EXP_INFORWARD_VALIDATE=1: every in-forward snapshot step also runs the two +# context iterations it replaces (the chunk up to P, then the tail) on copies of the pool +# rows, and prints how far the split's states and outputs are from theirs. Debug only. +# The split the executor's warmup runs for a dummy request (`_warm_inforward_split`) reports +# as `[inforward-validate] [warmup] ...`, so a line without `[warmup]` is a served request's. +_INFORWARD_VALIDATE = os.environ.get("TRTLLM_QWEN4_EXP_INFORWARD_VALIDATE", "0") == "1" +_inforward_validate_reports = 0 +_inforward_validate_warmup = False + + +def _inforward_report(what: str, got: torch.Tensor, ref: torch.Tensor) -> None: + global _inforward_validate_reports + if _inforward_validate_reports >= 400: + return + _inforward_validate_reports += 1 + g, r = got.float(), ref.float() + diff = float((g - r).abs().max()) if g.numel() else 0.0 + scale = float(r.abs().max()) if r.numel() else 0.0 + tag = "[warmup] " if _inforward_validate_warmup else "" + print( + f"[inforward-validate] {tag}{what}: bitwise={bool(torch.equal(got, ref))} " + f"max_abs_diff={diff:.3e} ref_max_abs={scale:.3e} shape={tuple(got.shape)}", + flush=True, + ) + + +def _inforward_reference_layout(split, num_tokens: int): + """The two iterations' row sets of a split step: the rows before every tail (`main`, + with their `cu`), and for each split request its index among the step's requests.""" + device = split.cu.device + keep = torch.ones(num_tokens, dtype=torch.bool, device=device) + keep[split.tail_tok] = False + main = torch.nonzero(keep).reshape(-1) + # Request ends in packed rows: the end of each request's last gather row. + num_rows = split.gather.shape[0] + is_main = torch.zeros(num_rows, dtype=torch.bool, device=device) + is_main[split.main_rows] = True + last = torch.ones(num_rows, dtype=torch.bool, device=device) + last[:-1] = is_main[1:] + ends = split.cu[1:][last] + tails = torch.zeros_like(ends) + request_of_split = torch.searchsorted(split.main_rows, split.split_rows) + tail_len = split.cu_tail[1:] - split.cu_tail[:-1] + tails[request_of_split] = tail_len + lens = torch.diff(ends, prepend=ends.new_zeros(1)) - tails + cu_main = torch.cat([lens.new_zeros(1), torch.cumsum(lens, 0)]) + return main, cu_main, request_of_split + + +class _InforwardSplit(NamedTuple): + """A context-only step's in-forward snapshot split (`Mamba2Metadata.inforward_split`, + planned by the cache manager): `k` requests run through their prompt end in this chunk + instead of taking one more context iteration for the tail past their snapshot point P. + The recurrent layers leave the state at P in the request's slot and stage the state at + the prompt end in the manager's staging row (`finish_inforward_snapshots`). + + `split` holds the long index tensors; the rest are this target's views of them: slot + vectors in the pool's index dtype, the carry flags of the sequences cut at P (a tail is a + sequence of its own that the first pass starts from zeros), and for the PLE layer the + rows before each P its conv state (`ple_conv_win`) and id context (`ple_ngram_win`) at P + are rebuilt from.""" + + split: NamedTuple + gather: torch.Tensor + carried: torch.Tensor + live: torch.Tensor + ple_conv_win: torch.Tensor + ple_ngram_win: torch.Tensor + # The staging rows in the pool's index dtype. + stage_idx: torch.Tensor + + +def _inforward_split(md, mm, si_all, hi_all, pools) -> _InforwardSplit | None: + """This step's in-forward snapshot split, or None on a step without one.""" + split = getattr(mm, "inforward_split", None) + if split is None: + return None + if _INFORWARD_VALIDATE: + global _inforward_validate_warmup + _inforward_validate_warmup = bool(getattr(mm, "_warmup_inforward_split", False)) + assert md.num_seqs == md.num_contexts, ( + "an in-forward snapshot split on a step with generation rows" + ) + carried = torch.zeros(split.gather.shape[0], dtype=hi_all.dtype, device=hi_all.device) + carried.index_copy_(0, split.main_rows, hi_all[: md.num_contexts]) + # The first packed row of each planned request's tail: its snapshot point P. + points = split.cu.index_select(0, split.split_rows + 1) + + def window(width: int) -> torch.Tensor: + back = torch.arange(width, 0, -1, dtype=points.dtype, device=points.device) + return torch.reshape(points[:, None] - back[None, :], [-1]) + + ple_conv, ple_ngram = pools.ple_state + return _InforwardSplit( + split=split, + gather=split.gather.to(si_all.dtype), + carried=carried, + live=split.live.to(si_all.dtype), + ple_conv_win=window(ple_conv.shape[-1]), + ple_ngram_win=window(ple_ngram.shape[-1]), + stage_idx=split.stage_rows.to(si_all.dtype), + ) + + class _PrefillStep(NamedTuple): """One step's host-side state, built by `PrefillTarget.prepare` and read by the forward's breaks. `num_tokens` is the batch's row count `n` and `physical_tokens` the `T` the @@ -3780,6 +3893,8 @@ class _PrefillStep(NamedTuple): fp8_pages: int | None = None # `_live_score_columns`: a bound on every row's selection window. live_cols: int | None = None + # A context-only step with in-forward snapshots: `_InforwardSplit`. + inforward: _InforwardSplit | None = None class PrefillTarget: @@ -4095,6 +4210,8 @@ def _gdn_core(self, proj, out, i) -> None: core.head_k_dim, out[0, num_prefill_tokens:num_tokens].view(num_decode_tokens, core.value_dim), ) + elif step.inforward is not None: + self._gdn_inforward(mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all, out, num_tokens) else: q, k, v, g, beta = self._gdn_prefill_prep( mixed_qkv, a, b, L, i, qsl_all, si_all, hi_all @@ -4121,6 +4238,138 @@ def _gdn_core(self, proj, out, i) -> None: ) out[:, num_tokens:].zero_() + def _gdn_inforward(self, mq, a, b, L, i, qsl, si, hi, out, num_tokens) -> None: + """The context-only recurrence of a step with in-forward snapshots (`_InforwardSplit`). + + The conv pass runs over whole chunks, so it leaves each planned slot the window that + ends at the prompt end: that goes to the staging row and the window that ends at P, + the raw inputs before P, into the slot. The recurrence is cut at P + (`flashinfer_chunk_gdn_inforward_split`).""" + core = self.core + inf = self._step.inforward + split = inf.split + pools = core._bound.pools + conv_states = pools.conv[i] + stage_ssm, stage_conv = core._bound.cm.inforward_snapshot_stage_views(i) + ref = ( + self._gdn_inforward_reference(mq, a, b, L, i, qsl, si, hi, num_tokens) + if _INFORWARD_VALIDATE + else None + ) + window = mq.index_select(0, split.conv_win) + q, k, v, g, beta = self._gdn_prefill_prep(mq, a, b, L, i, qsl, si, hi) + slots = split.tail_slots + stage_conv.index_copy_(0, split.stage_rows, conv_states.index_select(0, slots)) + conv_states.index_copy_( + 0, + slots, + torch.transpose(torch.reshape(window, [split.k, conv_states.shape[-1], -1]), 1, 2).to( + conv_states.dtype + ), + ) + flashinfer_chunk_gdn_inforward_split( + q, + k, + v, + g, + beta, + pools.ssm[i], + cu_seqlens=split.cu, + gather=inf.gather, + has_initial_state=inf.carried, + main_rows=split.main_rows, + live=inf.live, + tail_tok=split.tail_tok, + cu_tail=split.cu_tail, + tail_slots=split.tail_slots, + stage=stage_ssm, + stage_rows=split.stage_rows, + stage_idx=inf.stage_idx, + tail_lo=split.tail_lo, + tail_hi=split.tail_hi, + output=out[:, :num_tokens], + ) + if ref is not None: + ssm_p, conv_p, ssm_end, conv_end, out_ref, live, of_split = ref + _inforward_report( + f"gdn L{i} ssm@P (all requests)", pools.ssm[i].index_select(0, live), ssm_p + ) + _inforward_report( + f"gdn L{i} conv@P (all requests)", conv_states.index_select(0, live), conv_p + ) + rows = split.stage_rows + _inforward_report( + f"gdn L{i} ssm@end (split)", + stage_ssm.index_select(0, rows), + ssm_end.index_select(0, of_split), + ) + _inforward_report( + f"gdn L{i} conv@end (split)", + stage_conv.index_select(0, rows), + conv_end.index_select(0, of_split), + ) + _inforward_report(f"gdn L{i} output", out[0, :num_tokens], out_ref) + + def _gdn_inforward_reference(self, mq, a, b, L, i, qsl, si, hi, num_tokens): + """`_INFORWARD_VALIDATE`: the chunk up to P and then the tail, as two context + iterations run them, over copies of the step's pool rows (slot `r` = request `r`). + Returns the states at P, the states at the prompt end and the output rows.""" + core = self.core + split = self._step.inforward.split + pools = core._bound.pools + live = split.live + main, cu_main, of_split = _inforward_reference_layout(split, num_tokens) + ssm = pools.ssm[i].index_select(0, live).contiguous() + conv = pools.conv[i].index_select(0, live).contiguous() + idx = torch.arange(live.shape[0], dtype=si.dtype, device=si.device) + out_ref = torch.zeros( + num_tokens, core.num_v_heads, core.head_v_dim, dtype=mq.dtype, device=mq.device + ) + + def run(rows, cu, slots, carried): + qv = gdn_prefill_conv_post( + x=mq.index_select(0, rows).contiguous(), + weight=L.conv_w, + conv_states=conv, + query_start_loc=cu.to(qsl.dtype), + cache_indices=slots, + has_initial_state=carried, + a=a.index_select(0, rows).contiguous(), + b=b.index_select(0, rows).contiguous(), + A_log=L.a_log, + dt_bias=L.dt_bias, + num_k_heads=core.num_k_heads, + head_k_dim=core.head_k_dim, + num_v_heads=core.num_v_heads, + head_v_dim=core.head_v_dim, + pad_slot_id=_PAD_SLOT_ID, + l2_norm_eps=_L2_NORM_EPS, + softplus_threshold=_SOFTPLUS_THRESHOLD, + linear_gate=True, + ) + o, _ = flashinfer_chunk_gated_delta_rule( + *qv, + scale=None, + initial_state=ssm, + initial_state_indices=slots, + inplace_indexed_state_update=True, + output_final_state=False, + cu_seqlens=cu, + head_first=False, + use_qk_l2norm_in_kernel=False, + output=None, + state_workspace=None, + has_initial_state=carried, + linear_gate=True, + ) + out_ref.index_copy_(0, rows, o[0].to(out_ref.dtype)) + + run(main, cu_main, idx, hi) + ssm_p, conv_p = ssm.clone(), conv.clone() + tail_slots = of_split.to(si.dtype) + run(split.tail_tok, split.cu_tail, tail_slots, torch.ones_like(hi[: split.k])) + return ssm_p, conv_p, ssm, conv, out_ref, live, of_split + def _gdn_prefill_prep(self, mq, a, b, L, i, qsl, si, hi): """The context rows' recurrence inputs in one pass over `mq`, the token-major q|k|v column view of the projection: the causal conv against each sequence's carried @@ -4186,8 +4435,15 @@ def _ple_block(self, hc, hc_out, normed_out) -> None: num_tokens = step.num_tokens next_norm = self._layers[core.ple_layer].ahc_norm conv_input = None + inf = step.inforward + ple_ref = None + if inf is not None and _INFORWARD_VALIDATE: + ple_ref = self._ple_inforward_reference( + hc, conv_states, ngram_pool, si_all, carry_all, qsl + ) if step.stage: core.spec_rollback.stage_ple_verify(ngram_pool, conv_states, step.input_ids) + if step.stage or inf is not None: conv_input = torch.empty_like(hc[:num_tokens]) ngram = ple_ngram_embed( step.input_ids, @@ -4230,9 +4486,119 @@ def _ple_block(self, hc, hc_out, normed_out) -> None: ) if step.stage: core.spec_rollback.stage_ple_conv_input(conv_input) + if inf is not None: + # In-forward snapshots: both pools hold each planned slot's state at the prompt + # end. Stage it and rebuild the state at P from the rows before P: the conv + # input tail and the last ids. + split = inf.split + slots = split.tail_slots + stage_conv, stage_ngram = core._bound.cm.inforward_ple_stage_views(core.ple_layer) + stage_conv.index_copy_(0, split.stage_rows, conv_states.index_select(0, slots)) + stage_ngram.index_copy_(0, split.stage_rows, ngram_pool.index_select(0, slots)) + conv_states.index_copy_( + 0, + slots, + torch.transpose( + torch.reshape( + conv_input.index_select(0, inf.ple_conv_win), + [split.k, conv_states.shape[-1], -1], + ), + 1, + 2, + ).to(conv_states.dtype), + ) + ngram_pool.index_copy_( + 0, + slots, + torch.reshape(step.input_ids.index_select(0, inf.ple_ngram_win), [split.k, -1]).to( + ngram_pool.dtype + ), + ) + if ple_ref is not None: + conv_p, ngram_p, conv_end, ngram_end, hc_ref, normed_ref, live, of_split = ple_ref + rows = split.stage_rows + _inforward_report( + "ple conv@P (all requests)", conv_states.index_select(0, live), conv_p + ) + _inforward_report( + "ple ngram@P (all requests)", ngram_pool.index_select(0, live), ngram_p + ) + _inforward_report( + "ple conv@end (split)", + stage_conv.index_select(0, rows), + conv_end.index_select(0, of_split), + ) + _inforward_report( + "ple ngram@end (split)", + stage_ngram.index_select(0, rows), + ngram_end.index_select(0, of_split), + ) + _inforward_report("ple hc_out", hc_out[:num_tokens], hc_ref) + _inforward_report("ple normed", normed_out[:num_tokens], normed_ref) hc_out[num_tokens:].zero_() normed_out[num_tokens:].zero_() + def _ple_inforward_reference(self, hc, conv_states, ngram_pool, si_all, carry_all, qsl): + """`_INFORWARD_VALIDATE`: the PLE block over the chunk up to P and then over the tail, + as two context iterations run it, on copies of the step's pool rows.""" + core = self.core + S = self._stack + step = self._step + split = step.inforward.split + num_tokens = step.num_tokens + live = split.live + main, cu_main, of_split = _inforward_reference_layout(split, num_tokens) + conv = conv_states.index_select(0, live).contiguous() + ngram = ngram_pool.index_select(0, live).contiguous() + idx = torch.arange(live.shape[0], dtype=si_all.dtype, device=si_all.device) + next_norm = self._layers[core.ple_layer].ahc_norm + hc_ref = torch.zeros_like(hc[:num_tokens]) + normed_ref = torch.zeros_like(hc[:num_tokens]) + + def run(rows, cu, slots, carried): + n = rows.shape[0] + ids = step.input_ids.index_select(0, rows).contiguous() + cu = cu.to(qsl.dtype) + ng = ple_ngram_embed( + ids, ngram, slots, carried, cu, core.w["ple_table"], S.ple_tscale, S.ple_mult, + S.ple_vocab, S.ple_offs, core.eos_token_id, n, + row_window=_ple_window(core), + ) # fmt: skip + if _ple_needs_sum(core): + ng = core.tp_sum(ng) + o, nm = ple_gate_conv_inject_norm( + hc.index_select(0, rows).contiguous(), + cublas_mm(ng, S.ple_key), + cublas_mm(ng, S.ple_val), + S.ple_nk_w, + S.ple_nq_w, + S.ple_nc_w, + S.ple_conv_w, + conv, + slots, + cu, + next_norm, + core.eps, + core.hc_count, + _PLE_CLAMP_MIN, + core.ple_dilation, + _PAD_SLOT_ID, + n, + carried, + ) + hc_ref.index_copy_(0, rows, o) + normed_ref.index_copy_(0, rows, nm) + + run(main, cu_main, idx, carry_all) + conv_p, ngram_p = conv.clone(), ngram.clone() + run( + split.tail_tok, + split.cu_tail, + of_split.to(si_all.dtype), + torch.ones_like(carry_all[: split.k]), + ) + return conv_p, ngram_p, conv, ngram, hc_ref, normed_ref, live, of_split + def _splice(self, h) -> None: """Break: write the step's vision rows over their placeholder rows of the token embedding `h` `[T, hidden]` in place, and zero rows `[n:]`.""" @@ -4410,6 +4776,7 @@ def prepare( fp8_pages=fp8_pages, nvfp4_context=nvfp4_context, live_cols=_live_score_columns(core, md, batch), + inforward=_inforward_split(md, mm, si_all, hi_all, pools), ) def forward( @@ -6801,6 +7168,12 @@ class ModelingV2Qwen38FlashNextNvfp4Sm103Default( # decode-step prepare), validated there; off at every other layout. spec_decode_fast_paths: bool = False runtime_hooks: RuntimeHooks | None = None + # A context chunk that reaches the prompt end takes the Mamba snapshot before it inside + # the same forward: the GDN layers and the PLE layer split their state at the snapshot + # point (`PrefillTarget._gdn_inforward`, `_ple_block`). A positive value turns the path + # on in the cache manager; 1 leaves the fewest chunk tokens before the snapshot point at + # the manager's own floor (the GDN conv window and the PLE windows). + inforward_snapshot_min_main = 1 @classmethod def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py index 43e138a823e6..694c20e2e79d 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py @@ -385,6 +385,10 @@ class InforwardSnapshotSplit(NamedTuple): cu_tail: torch.Tensor tail_slots: torch.Tensor conv_win: torch.Tensor + # Host ints, set when exactly one request is split: its tail tokens are the + # packed rows [tail_lo, tail_hi), so a consumer can slice instead of gather. + tail_lo: int = -1 + tail_hi: int = -1 # Fields of InforwardSnapshotSplit, in upload order. @@ -686,8 +690,14 @@ def _prepare_inforward_split(self, attn_metadata: AttentionMetadata, name: self._inforward_split_buf[start:start + len(vals[name])] for name, start in zip(_INFORWARD_SPLIT_FIELDS, offsets) } + tail_lo = vals["tail_tok"][0] if k == 1 else -1 + tail_hi = vals["tail_tok"][-1] + 1 if k == 1 else -1 self.inforward_split = InforwardSnapshotSplit( - k=k, num_tail_tokens=num_tail_tokens, **views) + k=k, + num_tail_tokens=num_tail_tokens, + **views, + tail_lo=tail_lo, + tail_hi=tail_hi) @staticmethod def _warmup_inforward_split_plan(attn_metadata: AttentionMetadata, diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_flashinfer_chunk_gdn_inforward_split.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_flashinfer_chunk_gdn_inforward_split.py new file mode 100644 index 000000000000..0b123009d240 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/ops/ssm/test_modeling_v2_flashinfer_chunk_gdn_inforward_split.py @@ -0,0 +1,250 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU test for flashinfer_chunk_gdn_inforward_split (the in-forward Mamba snapshot split of +the Qwen3.8 GDN prefill). + +A context chunk that reaches the prompt end is cut at the snapshot point P inside one +forward. The test holds what the cache manager then relies on: + +- each split request's slot holds the state at P and its staging row the state at the + prompt end, an unsplit request's slot the state at its chunk end, and the outputs are + those of the two context iterations the split replaces (fp64 reference, the op's band); +- no other slot and no other staging row is written; +- a single split request's tail read and written as a row slice is bitwise the gathered + tail; +- `Mamba2Metadata` names the tail's row range only when exactly one request is split. +""" + +import types + +import pytest +import torch + +from tensorrt_llm._torch.modeling_v2.models.qwen4_exp.ops.ssm.flashinfer_chunk_gated_delta_rule import ( # noqa: E501 + INPLACE_MAX_TOKENS, + RANK4_GEOMETRY, + flashinfer_chunk_gdn_inforward_split, +) +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata +from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import InforwardSnapshotEntry + +assert torch.cuda.is_available(), "flashinfer_chunk_gdn_inforward_split requires a CUDA device" + +DEV = "cuda" +#: The band of flashinfer_chunk_gated_delta_rule against fp64 (its test's constants). +RTOL, ATOL_OUTPUT, ATOL_STATE = 1.6e-2, 2e-3, 2e-2 +NUM_SLOTS, STAGE_ROWS = 16, 4 +#: A stand-in for the conv window width: the split's `conv_win` is not read by this op. +_CONV_WIDTH = 3 + +# (request id, slot, chunk length, carries a state, tokens past P or None) +#: Three requests split (the gathered tail), two not; one of the split ones is fresh. +_MANY = ( + (11, 5, 40, True, None), + (12, 2, 70, True, 2), + (13, 7, 100, False, 2), + (14, 9, 33, False, None), + (15, 0, 130, True, 2), +) +#: Exactly one request split (the sliced tail), not the last row of the batch. +_ONE = ( + (21, 3, 90, True, None), + (22, 11, 200, True, 2), + (23, 6, 17, False, None), +) +#: A longer tail than the two tokens of `additional_snapshot_offsets_from_end: [2]`. +_LONG_TAIL = ( + (31, 4, 150, True, 7), + (32, 1, 64, True, None), +) + + +def _split(batch): + """`Mamba2Metadata.inforward_split` for `batch`, as the executor plans it.""" + plan, row = {}, 0 + for rid, _, length, _, after in batch: + if after is not None: + plan[rid] = InforwardSnapshotEntry( + row=row, point=length - after, num_before=length - after, num_after=after + ) + row += 1 + slots = {rid: slot for rid, slot, _, _, _ in batch} + manager = types.SimpleNamespace( + conv_state_shape=[8, _CONV_WIDTH], + inforward_snapshot_rows=STAGE_ROWS, + inforward_snapshot_max_tail=16, + inforward_snapshot_plan=plan, + state_index=lambda rid: slots[rid], + ) + attn = types.SimpleNamespace( + request_ids=[rid for rid, _, _, _, _ in batch], + seq_lens=torch.tensor([length for _, _, length, _, _ in batch], dtype=torch.int32), + ) + metadata = Mamba2Metadata(max_batch_size=8, chunk_size=128) + metadata._prepare_inforward_split(attn, manager, plan, len(batch), len(batch)) + return metadata.inforward_split + + +def _inputs(batch, seed): + """Post-conv GDN operands (unit-norm q/k, the linear gate, beta a sigmoid), a bf16 pool + and staging rows of random bits, so a row the op should not write is seen if it is.""" + gen = torch.Generator(device=DEV).manual_seed(seed) + num_k_heads, num_v_heads, head_k, head_v = RANK4_GEOMETRY + total = sum(length for _, _, length, _, _ in batch) + q = torch.randn(1, total, num_k_heads, head_k, generator=gen, device=DEV) + k = torch.randn(1, total, num_k_heads, head_k, generator=gen, device=DEV) + q = (q / torch.sqrt(q.pow(2).sum(-1, keepdim=True) + 1e-6)).bfloat16() + k = (k / torch.sqrt(k.pow(2).sum(-1, keepdim=True) + 1e-6)).bfloat16() + v = torch.randn(1, total, num_v_heads, head_v, generator=gen, device=DEV).bfloat16() + a_log = torch.log(torch.rand(num_v_heads, generator=gen, device=DEV) * 15 + 1) + x = torch.randn(1, total, num_v_heads, generator=gen, device=DEV) + alpha = torch.exp(-torch.exp(a_log) * torch.nn.functional.softplus(x)).float() + beta = torch.rand(1, total, num_v_heads, generator=gen, device=DEV).sigmoid().float() + + def states(rows): + return ( + torch.randn(rows, num_v_heads, head_v, head_k, generator=gen, device=DEV) * 0.1 + ).bfloat16() + + return (q, k, v, alpha, beta), states(NUM_SLOTS), states(STAGE_ROWS) + + +def _run(batch, operands, pool, stage, sliced=True): + """The op as `PrefillTarget._gdn_core` calls it, on copies of `pool` and `stage`. + + With `sliced` false it names no tail row range, so a single split request's tail is + gathered like several requests' tails.""" + split = _split(batch) + q, k, v, alpha, beta = operands + pool, stage = pool.clone(), stage.clone() + index = torch.int32 + carried = torch.zeros(split.gather.shape[0], dtype=torch.bool, device=DEV) + carried.index_copy_( + 0, + split.main_rows, + torch.tensor([carries for _, _, _, carries, _ in batch], dtype=torch.bool, device=DEV), + ) + output = torch.full( + (1, q.shape[1], v.shape[2], v.shape[3]), -7.0, dtype=torch.bfloat16, device=DEV + ) + lo, hi = (split.tail_lo, split.tail_hi) if sliced else (-1, -1) + flashinfer_chunk_gdn_inforward_split( + q, + k, + v, + alpha, + beta, + pool, + cu_seqlens=split.cu, + gather=split.gather.to(index), + has_initial_state=carried, + main_rows=split.main_rows, + live=split.live.to(index), + tail_tok=split.tail_tok, + cu_tail=split.cu_tail, + tail_slots=split.tail_slots, + stage=stage, + stage_rows=split.stage_rows, + stage_idx=split.stage_rows.to(index), + output=output, + tail_lo=lo, + tail_hi=hi, + ) + torch.cuda.synchronize() + return output, pool, stage + + +def _scan(operands, state, start, end): + """fp64 token-at-a-time recurrence over rows `[start, end)` from `state`: + `(outputs, final state)`.""" + q, k, v, alpha, beta = operands + dt = torch.float64 + repeats = v.shape[2] // q.shape[2] + scale = q.shape[3] ** -0.5 + outs = [] + for t in range(start, end): + q_t = q[0, t].to(dt).repeat_interleave(repeats, dim=0) * scale + k_t = k[0, t].to(dt).repeat_interleave(repeats, dim=0) + state = state * alpha[0, t].to(dt)[:, None, None] + u = (v[0, t].to(dt) - (state * k_t[:, None, :]).sum(-1)) * beta[0, t].to(dt)[:, None] + state = state + u[:, :, None] * k_t[:, None, :] + outs.append((state * q_t[:, None, :]).sum(-1)) + return torch.stack(outs), state + + +def _two_iterations(batch, operands, pool, stage): + """What two context iterations leave: the chunk up to P into the slot (rounded to the + pool's dtype, as the first iteration's scatter does), then the tail from that state.""" + out = torch.empty( + 1, operands[0].shape[1], operands[2].shape[2], operands[2].shape[3], dtype=torch.float64 + ).to(DEV) + pool, stage = pool.clone(), stage.clone() + start, row = 0, 0 + for _, slot, length, carries, after in batch: + before = length if after is None else length - after + state = pool[slot].to(torch.float64) + if not carries: + state = torch.zeros_like(state) + out[0, start : start + before], state = _scan(operands, state, start, start + before) + pool[slot] = state.to(pool.dtype) + if after is not None: + out[0, start + before : start + length], state = _scan( + operands, pool[slot].to(torch.float64), start + before, start + length + ) + stage[row] = state.to(stage.dtype) + row += 1 + start += length + return out.to(torch.bfloat16), pool, stage + + +@pytest.mark.parametrize( + "batch", [_MANY, _ONE, _LONG_TAIL], ids=["three_split", "one_split", "long_tail"] +) +def test_split_is_the_two_context_iterations(batch) -> None: + operands, pool, stage = _inputs(batch, seed=1) + out, got_pool, got_stage = _run(batch, operands, pool, stage) + want_out, want_pool, want_stage = _two_iterations(batch, operands, pool, stage) + + torch.testing.assert_close(out, want_out, rtol=RTOL, atol=ATOL_OUTPUT) + torch.testing.assert_close(got_pool, want_pool, rtol=RTOL, atol=ATOL_STATE) + torch.testing.assert_close(got_stage, want_stage, rtol=RTOL, atol=ATOL_STATE) + # The slots of other requests and the staging rows no request was given are not written. + named = {slot for _, slot, _, _, _ in batch} + others = [slot for slot in range(NUM_SLOTS) if slot not in named] + assert torch.equal(got_pool[others], pool[others]) + num_split = sum(after is not None for _, _, _, _, after in batch) + assert torch.equal(got_stage[num_split:], stage[num_split:]) + # The state at P is not the state at the prompt end: a split that left the wrong one in + # the slot is outside the band. + split_slots = [slot for _, slot, _, _, after in batch if after is not None] + assert not torch.allclose( + got_pool[split_slots].float(), + want_stage[:num_split].float(), + rtol=RTOL, + atol=ATOL_STATE, + ) + + +@pytest.mark.parametrize("batch", [_ONE, _LONG_TAIL], ids=["two_token_tail", "long_tail"]) +def test_sliced_tail_is_bitwise_the_gathered_tail(batch) -> None: + operands, pool, stage = _inputs(batch, seed=2) + gathered = _run(batch, operands, pool, stage, sliced=False) + sliced = _run(batch, operands, pool, stage, sliced=True) + for name, got, want in zip(("output", "pool", "stage"), sliced, gathered): + assert torch.equal(got, want), name + + +def test_tail_row_range_is_named_for_a_single_split_request_only() -> None: + one = _split(_ONE) + assert one.k == 1 + # Request 22's last two rows: it follows a 90-token request and is 200 tokens long. + assert (one.tail_lo, one.tail_hi) == (90 + 198, 90 + 200) + assert one.tail_tok.tolist() == list(range(one.tail_lo, one.tail_hi)) + + many = _split(_MANY) + assert many.k == 3 + assert (many.tail_lo, many.tail_hi) == (-1, -1) + + long_tail = _split(_LONG_TAIL) + assert (long_tail.tail_lo, long_tail.tail_hi) == (150 - 7, 150) + assert long_tail.num_tail_tokens == 7 <= INPLACE_MAX_TOKENS diff --git a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/ssm/test_modeling_v2_gdn_block.py b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/ssm/test_modeling_v2_gdn_block.py index d124f9d9b04e..2b53316cede4 100644 --- a/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/ssm/test_modeling_v2_gdn_block.py +++ b/tests/unittest/_torch/modeling_v2/models/qwen4_exp/qwen3_8_flash_next_nvfp4__sm_103__default/ssm/test_modeling_v2_gdn_block.py @@ -284,7 +284,9 @@ def test_captured_one_rank_fp32_decode_block() -> None: def _prefill(core, L, x, step): tokens = x.shape[0] out = torch.empty([1, tokens, core.num_v_heads, V], dtype=BF16, device=DEV) - target = _target(M.PrefillTarget, core, L, types.SimpleNamespace(gdn=step, num_tokens=tokens)) + target = _target( + M.PrefillTarget, core, L, types.SimpleNamespace(gdn=step, num_tokens=tokens, inforward=None) + ) return target._gdn_block(x, 0, L, out)