Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ def forward(
pre_aux: Optional[
Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]
] = None,
decode_topk_start_event: Optional[torch.cuda.Event] = None,
):
use_overlapped_prepare = (
do_multi_stream()
Expand Down Expand Up @@ -404,5 +405,6 @@ def forward(
k_scale,
weights,
q_scale=q_scale,
decode_topk_start_event=decode_topk_start_event,
)
return topk_indices
Original file line number Diff line number Diff line change
Expand Up @@ -1446,34 +1446,35 @@ def _compressor_branch():
self.compressor(hidden_states, attn_metadata)
return None

def _indexer_branch():
def _indexer_branch(decode_topk_start_event: Optional[torch.cuda.Event] = None):
return self.indexer(
qr,
hidden_states,
attn_metadata,
position_ids,
pre_aux=_indexer_pre_aux,
decode_topk_start_event=decode_topk_start_event,
)

topk_indices = None
indexer_ran = False
if _v4_extra_overlap:
if _use_indexer_overlap:
# Compressor and indexer prework are already in flight. Run Q after the
# compressor on the same stream, then synchronize both streams below.
# Issue Q first: decode Top-K waits on dsv4_compressor_event, so its
# record must be enqueued before the indexer consumes the event.
self.dsv4_overlap_start_event.record()

with torch.cuda.stream(self.indexer_stream):
self.dsv4_overlap_start_event.wait()
topk_indices = _indexer_branch()
indexer_ran = True
self.dsv4_indexer_event.record()

with torch.cuda.stream(self.compressor_stream):
self.dsv4_overlap_start_event.wait()
q = _q_branch()
self.dsv4_compressor_event.record()

with torch.cuda.stream(self.indexer_stream):
Comment thread
trtllm-agent marked this conversation as resolved.
Resolved
self.dsv4_overlap_start_event.wait()
topk_indices = _indexer_branch(self.dsv4_compressor_event)
Comment thread
trtllm-agent marked this conversation as resolved.
Resolved
Comment thread
trtllm-agent marked this conversation as resolved.
Resolved
indexer_ran = True
self.dsv4_indexer_event.record()

self.dsv4_compressor_event.wait()
self.dsv4_indexer_event.wait()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2012,13 +2012,18 @@ def sparse_attn_indexer(
weights: torch.Tensor,
q_scale: Optional[torch.Tensor] = None,
is_generation: Optional[bool] = None,
decode_topk_start_event: Optional[torch.cuda.Event] = None,
) -> torch.Tensor:
"""Run the indexer TopK kernel for one phase or the full batch.

When ``is_generation`` is ``None``, the inputs contain the full mixed
batch. Otherwise they contain only the selected context or generation
phase. ``q_scale`` is only consumed by the FP4 dispatch.

``decode_topk_start_event`` is waited on the current stream immediately
before decode Top-K, after logits computation. It does not delay prefill
Top-K, and its producer must enqueue the record before this call.

"""
# DSACacheManager / DeepseekV4CacheManager force quant_block_size to
# 128 (FP8 path) or 32 (MXFP4 path); both round-trip to the same
Expand Down Expand Up @@ -2483,6 +2488,8 @@ def sparse_attn_indexer(
)
assert metadata.radix_aux_indices is not None
assert metadata.radix_aux_logits is not None
if decode_topk_start_event is not None:
decode_topk_start_event.wait()
Comment thread
trtllm-agent marked this conversation as resolved.
Resolved
self.top_k(
logits_decode,
topk_indices_buffer[token_offset : token_offset + num_gen_tokens, :],
Expand Down
1 change: 0 additions & 1 deletion tests/integration/test_lists/waives.txt
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_
full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570)
full:GB200/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py::test_attention_dp_decoder_bounded_replay[world2] SKIP (https://nvbugs/7049813)
full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6661948)
full:GB300/perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] SKIP (https://nvbugs/7109962)
full:GB300/unittest/_torch/modeling/test_modeling_gpt_oss.py::test_gpt_oss_trtllmgen[cutlass] SKIP (https://nvbugs/6633932)
full:GB300/unittest/_torch/modules/tests_lora_modules/test_qwen3_sanity.py::TestQwen3LoRA::test_qwen3_fp8_lora SKIP (https://nvbugs/6668777)
full:H100/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/7063452)
Expand Down
Loading
Loading