diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/indexer.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/indexer.py index 995fc1c96ac8..e3c90ecd1d99 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/indexer.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/indexer.py @@ -372,6 +372,7 @@ def forward( pre_aux: Optional[ Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]] ] = None, + decode_topk_start_event: Optional[torch.cuda.Event] = None, ): use_overlapped_prepare = ( do_multi_stream() @@ -404,5 +405,6 @@ def forward( k_scale, weights, q_scale=q_scale, + decode_topk_start_event=decode_topk_start_event, ) return topk_indices diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py index 99ddbe62e999..5a5778b868c7 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py @@ -1446,34 +1446,35 @@ def _compressor_branch(): self.compressor(hidden_states, attn_metadata) return None - def _indexer_branch(): + def _indexer_branch(decode_topk_start_event: Optional[torch.cuda.Event] = None): return self.indexer( qr, hidden_states, attn_metadata, position_ids, pre_aux=_indexer_pre_aux, + decode_topk_start_event=decode_topk_start_event, ) topk_indices = None indexer_ran = False if _v4_extra_overlap: if _use_indexer_overlap: - # Compressor and indexer prework are already in flight. Run Q after the - # compressor on the same stream, then synchronize both streams below. + # Issue Q first: decode Top-K waits on dsv4_compressor_event, so its + # record must be enqueued before the indexer consumes the event. self.dsv4_overlap_start_event.record() - with torch.cuda.stream(self.indexer_stream): - self.dsv4_overlap_start_event.wait() - topk_indices = _indexer_branch() - indexer_ran = True - self.dsv4_indexer_event.record() - with torch.cuda.stream(self.compressor_stream): self.dsv4_overlap_start_event.wait() q = _q_branch() self.dsv4_compressor_event.record() + with torch.cuda.stream(self.indexer_stream): + self.dsv4_overlap_start_event.wait() + topk_indices = _indexer_branch(self.dsv4_compressor_event) + indexer_ran = True + self.dsv4_indexer_event.record() + self.dsv4_compressor_event.wait() self.dsv4_indexer_event.wait() diff --git a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py index e48d56a97d63..cc11eb5c5ba8 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/dsa/indexer.py @@ -2012,6 +2012,7 @@ def sparse_attn_indexer( weights: torch.Tensor, q_scale: Optional[torch.Tensor] = None, is_generation: Optional[bool] = None, + decode_topk_start_event: Optional[torch.cuda.Event] = None, ) -> torch.Tensor: """Run the indexer TopK kernel for one phase or the full batch. @@ -2019,6 +2020,10 @@ def sparse_attn_indexer( batch. Otherwise they contain only the selected context or generation phase. ``q_scale`` is only consumed by the FP4 dispatch. + ``decode_topk_start_event`` is waited on the current stream immediately + before decode Top-K, after logits computation. It does not delay prefill + Top-K, and its producer must enqueue the record before this call. + """ # DSACacheManager / DeepseekV4CacheManager force quant_block_size to # 128 (FP8 path) or 32 (MXFP4 path); both round-trip to the same @@ -2483,6 +2488,8 @@ def sparse_attn_indexer( ) assert metadata.radix_aux_indices is not None assert metadata.radix_aux_logits is not None + if decode_topk_start_event is not None: + decode_topk_start_event.wait() self.top_k( logits_decode, topk_indices_buffer[token_offset : token_offset + num_gen_tokens, :], diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c72ec0c71369..ea6441fcc157 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -103,7 +103,6 @@ full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_ full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:GB200/unittest/_torch/multi_gpu/test_deepseek_v41_attention_dp.py::test_attention_dp_decoder_bounded_replay[world2] SKIP (https://nvbugs/7049813) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6661948) -full:GB300/perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] SKIP (https://nvbugs/7109962) full:GB300/unittest/_torch/modeling/test_modeling_gpt_oss.py::test_gpt_oss_trtllmgen[cutlass] SKIP (https://nvbugs/6633932) full:GB300/unittest/_torch/modules/tests_lora_modules/test_qwen3_sanity.py::TestQwen3LoRA::test_qwen3_fp8_lora SKIP (https://nvbugs/6668777) full:H100/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/7063452) diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py index 36d56cd85e0d..1500224121a1 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py @@ -39,10 +39,16 @@ DeepseekV4TrtllmAttention, DeepseekV4TrtllmAttentionMetadata, ) +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.indexer import DeepseekV4Indexer +from tensorrt_llm._torch.attention.backends.sparse.deepseek_v4.params import DeepSeekV4Params from tensorrt_llm._torch.attention.backends.sparse.params import SparseBackendForwardArgs +from tensorrt_llm._torch.attention.mla import MLA from tensorrt_llm._torch.metadata import KVCacheParams +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modules.multi_stream_utils import with_multi_stream from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm._torch.utils import AuxStreamType from tensorrt_llm.bindings import DataType, SamplingConfig from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp from tensorrt_llm.functional import PositionEmbeddingType, RopeEmbeddingUtils @@ -669,6 +675,294 @@ def _create_pos_embd_params(scenario: Scenario) -> PositionalEmbeddingParams: ) +@skip_pre_blackwell +@pytest.mark.parametrize("phase", ["prefill", "decode"]) +def test_deepseek_v4_indexer_decode_start_event(monkeypatch, phase: str): + """Forward the real CUDA event to decode selection without delaying logits or prefill.""" + scenario = Scenario(hidden_size=128, q_lora_rank=32, max_position_embeddings=4096) + context_lengths = [2048] + cache_manager, sparse_config = _create_cache_manager(scenario, context_lengths, 4096) + + class DecodeStartEvent(torch.cuda.Event): + def __init__(self): + super().__init__() + self.was_waited = False + + def wait(self, stream=None): + self.was_waited = True + return super().wait(stream) + + try: + request = LlmRequest( + request_id=0, + max_new_tokens=2, + input_tokens=list(range(context_lengths[0])), + sampling_config=SamplingConfig(), + is_streaming=False, + ) + scheduled_batch = ScheduledRequests() + scheduled_batch.append_context_request(request) + cache_manager.prepare_context(request) + cache_manager.resize_context(request, request.context_chunk_size) + indexer = DeepseekV4Indexer( + quant_config=None, + pos_embd_params=_create_pos_embd_params(scenario), + mla_params=MLAParams( + q_lora_rank=scenario.q_lora_rank, + qk_rope_head_dim=scenario.qk_rope_head_dim, + hidden_size=scenario.hidden_size, + ), + skip_create_weights_in_init=False, + sparse_params=DeepSeekV4Params( + index_n_heads=64, + index_head_dim=128, + index_topk=scenario.index_topk, + indexer_k_dtype=sparse_config.indexer_k_dtype, + compress_ratios=scenario.compress_ratios, + skip_indexer_for_short_seqs=False, + use_cute_dsl_paged_mqa_logits=False, + ), + dtype=scenario.dtype, + compress_ratio=4, + layer_idx=1, + ).cuda() + with torch.no_grad(): + for parameter in indexer.parameters(): + parameter.zero_() + + metadata = DeepseekV4TrtllmAttentionMetadata( + seq_lens=torch.tensor(context_lengths, dtype=torch.int32), + request_ids=[0], + max_num_requests=1, + num_contexts=1, + prompt_lens=context_lengths, + max_num_tokens=context_lengths[0], + kv_cache_manager=cache_manager, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]), + mapping=Mapping(world_size=1, rank=0, tp_size=1), + sparse_attention_config=sparse_config, + ) + metadata.prepare() + qr = torch.zeros( + (context_lengths[0], scenario.q_lora_rank), dtype=scenario.dtype, device="cuda" + ) + hidden_states = torch.zeros( + (context_lengths[0], scenario.hidden_size), dtype=scenario.dtype, device="cuda" + ) + position_ids = torch.arange(context_lengths[0], dtype=torch.int32, device="cuda") + with torch.no_grad(): + indexer(qr, hidden_states, metadata, position_ids) + + if phase == "decode": + request.context_current_position = context_lengths[0] + request.add_new_token(context_lengths[0], 0) + cache_manager.update_context_resources(scheduled_batch) + _allocate_kv_cache_for_generation(cache_manager, [request], num_new_tokens=1) + metadata = DeepseekV4TrtllmAttentionMetadata( + seq_lens=torch.tensor([1], dtype=torch.int32), + request_ids=[0], + max_num_requests=1, + num_contexts=0, + prompt_lens=context_lengths, + max_num_tokens=1, + kv_cache_manager=cache_manager, + kv_cache_params=KVCacheParams( + use_cache=True, num_cached_tokens_per_seq=context_lengths + ), + mapping=Mapping(world_size=1, rank=0, tp_size=1), + sparse_attention_config=sparse_config, + ) + metadata.prepare() + qr = qr[:1] + hidden_states = hidden_states[:1] + position_ids = torch.tensor(context_lengths, dtype=torch.int32, device="cuda") + + event = DecodeStartEvent() + event.record() + selection_phases = set() + original_topk = indexer.top_k.forward + original_decode_logits = indexer._call_paged_mqa_logits + + def observed_decode_logits(*args, **kwargs): + assert not event.was_waited, "Q completion must not delay indexer logits" + return original_decode_logits(*args, **kwargs) + + def observed_topk(*args, **kwargs): + is_prefill = kwargs["is_prefill"] + assert event.was_waited == (not is_prefill) + selection_phases.add(is_prefill) + return original_topk(*args, **kwargs) + + monkeypatch.setattr(indexer, "_call_paged_mqa_logits", observed_decode_logits) + monkeypatch.setattr(indexer.top_k, "forward", observed_topk) + with torch.no_grad(): + result = indexer( + qr, hidden_states, metadata, position_ids, decode_topk_start_event=event + ) + torch.cuda.synchronize() + + assert selection_phases == {phase == "prefill"} + assert event.was_waited == (phase == "decode") + assert result.shape == (hidden_states.shape[0], scenario.index_topk) + assert (result >= 0).any() + finally: + cache_manager.shutdown() + + +@skip_pre_blackwell +def test_deepseek_v4_module_decode_start_event(monkeypatch: pytest.MonkeyPatch) -> None: + """Enqueue Q and its completion record before dispatching the overlapping indexer.""" + monkeypatch.setenv("TRTLLM_MLA_EXTRA_OVERLAP", "1") + monkeypatch.setenv("TRTLLM_MLA_Q_B_PROJ_USE_CUTE_DSL", "0") + scenario = Scenario( + hidden_size=128, + num_heads=16, + q_lora_rank=64, + qk_nope_head_dim=448, + max_position_embeddings=4096, + ) + context_lengths = [2048] + cache_manager, sparse_config = _create_cache_manager(scenario, context_lengths, 4096) + try: + mapping = Mapping(world_size=1, rank=0, tp_size=1) + mla = MLA( + hidden_size=scenario.hidden_size, + num_attention_heads=scenario.num_heads, + num_key_value_heads=scenario.num_kv_heads, + qk_nope_head_dim=scenario.qk_nope_head_dim, + qk_rope_head_dim=scenario.qk_rope_head_dim, + v_head_dim=scenario.v_head_dim, + q_lora_rank=scenario.q_lora_rank, + kv_lora_rank=scenario.kv_lora_rank - scenario.qk_rope_head_dim, + predicted_tokens_per_seq=1, + max_position_embeddings=scenario.max_position_embeddings, + bias=False, + aux_stream_dict={ + role: torch.cuda.Stream() + for role in ( + AuxStreamType.Attention, + AuxStreamType.MlaCompressor, + AuxStreamType.MlaIndexer, + AuxStreamType.MlaIndexerAux, + ) + }, + pos_embd_params=_create_pos_embd_params(scenario), + layer_idx=1, + dtype=scenario.dtype, + config=ModelConfig(mapping=mapping, sparse_attention_config=sparse_config), + num_groups=1, + o_lora_rank=128, + ).cuda() + with torch.no_grad(): + for parameter in mla.parameters(): + parameter.zero_() + + request = LlmRequest( + request_id=0, + max_new_tokens=2, + input_tokens=list(range(context_lengths[0])), + sampling_config=SamplingConfig(), + is_streaming=False, + ) + scheduled_batch = ScheduledRequests() + scheduled_batch.append_context_request(request) + cache_manager.prepare_context(request) + cache_manager.resize_context(request, request.context_chunk_size) + metadata = DeepseekV4TrtllmAttentionMetadata( + seq_lens=torch.tensor(context_lengths, dtype=torch.int32), + request_ids=[0], + max_num_requests=1, + num_contexts=1, + prompt_lens=context_lengths, + max_num_tokens=context_lengths[0], + kv_cache_manager=cache_manager, + kv_cache_params=KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]), + mapping=mapping, + sparse_attention_config=sparse_config, + ) + metadata.prepare() + hidden_states = torch.zeros( + (context_lengths[0], scenario.hidden_size), dtype=scenario.dtype, device="cuda" + ) + position_ids = torch.arange(context_lengths[0], dtype=torch.int32, device="cuda") + output = torch.empty( + (context_lengths[0], scenario.num_heads * scenario.v_head_dim), + dtype=scenario.dtype, + device="cuda", + ) + with torch.no_grad(): + mla.forward_impl(position_ids, hidden_states, metadata, [output]) + + request.context_current_position = context_lengths[0] + request.add_new_token(context_lengths[0], 0) + cache_manager.update_context_resources(scheduled_batch) + _allocate_kv_cache_for_generation(cache_manager, [request], num_new_tokens=1) + metadata = DeepseekV4TrtllmAttentionMetadata( + seq_lens=torch.tensor([1], dtype=torch.int32), + request_ids=[0], + max_num_requests=1, + num_contexts=0, + prompt_lens=context_lengths, + max_num_tokens=1, + kv_cache_manager=cache_manager, + kv_cache_params=KVCacheParams( + use_cache=True, num_cached_tokens_per_seq=context_lengths + ), + mapping=mapping, + sparse_attention_config=sparse_config, + ) + metadata.prepare() + hidden_states = hidden_states[:1] + position_ids = torch.tensor(context_lengths, dtype=torch.int32, device="cuda") + output = output[:1] + q_projected = False + indexer_entered = False + + class QCompletionEvent(torch.cuda.Event): + def __init__(self) -> None: + super().__init__() + self.was_recorded = False + + def record(self, stream: torch.cuda.Stream | None = None) -> None: + assert q_projected, "Q must be enqueued before its completion record" + result = super().record(stream) + self.was_recorded = True + return result + + producer = QCompletionEvent() + original_q_projection = mla.q_b_proj.forward + original_indexer = mla.indexer.forward + + def observed_q_projection(*args, **kwargs) -> torch.Tensor: + nonlocal q_projected + result = original_q_projection(*args, **kwargs) + q_projected = True + return result + + def observed_indexer(*args, **kwargs) -> torch.Tensor: + nonlocal indexer_entered + assert q_projected, "Q must be enqueued before indexer entry" + assert producer.was_recorded, "Q completion must be recorded before indexer entry" + assert kwargs.get("decode_topk_start_event") is producer, ( + "The indexer must receive the same Q completion event" + ) + indexer_entered = True + return original_indexer(*args, **kwargs) + + monkeypatch.setattr(mla, "dsv4_compressor_event", producer) + monkeypatch.setattr(mla.q_b_proj, "forward", observed_q_projection) + monkeypatch.setattr(mla.indexer, "forward", observed_indexer) + with torch.no_grad(), with_multi_stream(True): + mla.forward_impl(position_ids, hidden_states, metadata, [output]) + torch.cuda.synchronize() + + assert indexer_entered, "The decode forward must exercise indexer overlap" + assert torch.isfinite(output).all() + finally: + torch.cuda.synchronize() + cache_manager.shutdown() + + @skip_pre_blackwell def test_deepseek_v4_sparse_mla_single_token_tp4_local_heads_repro(): """Reproduce the tp=4 Flash warmup sparse-MLA kernel shape.