Repository navigation
[TRTLLM-17005][refactor] migrate attention workspace tests to a host sizing component #22457
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5b8d625
9b9dcbb
613efa3
13223a5
18d1d0f
0bffb8c
f418c79
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /* | ||
| * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include "tensorrt_llm/common/attentionWorkspace.h" | ||
|
|
||
| TRTLLM_NAMESPACE_BEGIN | ||
|
|
||
| namespace common::op | ||
| { | ||
|
|
||
| AttentionContextWorkspaceSizes AttentionWorkspaceManager::buildUnfusedContextSizes( | ||
| AttentionUnfusedContextWorkspaceParams const& params) noexcept | ||
| { | ||
| AttentionContextWorkspaceSizes sizes{}; | ||
| auto const paddedQueryTokens = params.batchSize * params.querySequenceLength; | ||
| auto const paddedKvTokens = params.batchSize * params.kvSequenceLength; | ||
| auto const queryHiddenUnits = params.numQueryHeads * params.headSize; | ||
| auto const kvHiddenUnits = params.numAttnKvHeads * params.headSize; | ||
| auto const scoreElements = paddedQueryTokens * params.numQueryHeads * params.kvSequenceLength; | ||
| sizes.attentionMask = params.elementSize * paddedQueryTokens * params.kvSequenceLength; | ||
| sizes.cuQSeqlens = sizeof(int) * (params.batchSize + 1); | ||
| sizes.cuKvSeqlens = sizes.cuQSeqlens; | ||
| sizes.cuMaskRows = sizes.cuQSeqlens; | ||
| sizes.qBuf = params.elementSize * paddedQueryTokens * queryHiddenUnits; | ||
| sizes.kBuf = params.elementSize * paddedKvTokens * kvHiddenUnits; | ||
| sizes.vBuf = sizes.kBuf; | ||
| sizes.qkBuf = params.elementSize * scoreElements; | ||
| sizes.qkvBuf = sizes.qBuf; | ||
| sizes.qkFloatBuf = sizeof(float) * scoreElements; | ||
| sizes.paddingOffset = sizeof(int) * paddedQueryTokens; | ||
| sizes.encoderPaddingOffset = sizeof(int) * params.batchSize * params.crossKvLength; | ||
| sizes.tokensInfo = sizeof(int2) * params.packedTokenCount; | ||
| return sizes; | ||
| } | ||
|
|
||
| } // namespace common::op | ||
|
|
||
| TRTLLM_NAMESPACE_END |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -802,32 +802,24 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL | |
|
|
||
| auto const batch_size = static_cast<size_t>(max_num_seq); | ||
| auto const kv_seq_length = (isCrossAttention(p) ? cross_kv_length : input_seq_length); | ||
| // The unfused-MHA buffers below must upper-bound the enqueueContext carve, which sizes them by | ||
| // batch_size * input_seq_length (not num_tokens): with padding removal the actual token count can be | ||
| // smaller than batch_size * max(context q length), so sizing by max_num_tokens underestimates. | ||
| size_t const attention_mask_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_length * kv_seq_length; | ||
| auto workspaceSizes = mEnableContextFMHA ? AttentionContextWorkspaceSizes{} | ||
| : AttentionWorkspaceManager::buildUnfusedContextSizes({.elementSize = size, | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Nice to have] (R1: PR process) This hunk overlaps open PR #21915 ("Phased FMHA libraries, selected in Python"). At its head c64c037, |
||
| .batchSize = batch_size, | ||
| .querySequenceLength = static_cast<size_t>(input_seq_length), | ||
| .kvSequenceLength = static_cast<size_t>(kv_seq_length), | ||
| .crossKvLength = static_cast<size_t>(cross_kv_length), | ||
| .packedTokenCount = static_cast<size_t>(max_num_tokens), | ||
| .numQueryHeads = mNumAttnHeads, | ||
| .numAttnKvHeads = mNumAttnKVHeads, | ||
| .headSize = getHeadSize()}); | ||
| size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1); | ||
| size_t const rotary_inv_freq_size = sizeof(float) * batch_size * rotary_embedding_dim / 2; | ||
|
|
||
| size_t q_buf_2_size = 0; | ||
| if (!mEnableContextFMHA) | ||
| { | ||
| // Unfused mha | ||
| q_buf_2_size = size * batch_size * input_seq_length * local_hidden_units_qo; | ||
| } | ||
| else if (mFmhaDispatcher->isSeparateQAndKvInput()) | ||
| if (mEnableContextFMHA && mFmhaDispatcher->isSeparateQAndKvInput()) | ||
| { | ||
| // Paged context fmha | ||
| q_buf_2_size = (mFP8ContextFMHA ? 1 : size) * max_num_tokens * local_hidden_units_qo; | ||
| workspaceSizes.qBuf = (mFP8ContextFMHA ? 1 : size) * max_num_tokens * local_hidden_units_qo; | ||
| } | ||
|
|
||
| size_t const k_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * kv_seq_length * local_hidden_units_kv; | ||
| size_t const v_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * kv_seq_length * local_hidden_units_kv; | ||
| size_t const qk_buf_size | ||
| = mEnableContextFMHA ? 0 : size * batch_size * num_heads * input_seq_length * kv_seq_length; | ||
| size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_length * local_hidden_units_qo; | ||
| size_t const qk_buf_float_size | ||
| = mEnableContextFMHA ? 0 : sizeof(float) * batch_size * num_heads * input_seq_length * kv_seq_length; | ||
| bool const useSageAttnSeparateQkv = mEnableContextFMHA && useSageAttn() && mFmhaDispatcher->isSeparateQAndKvInput(); | ||
|
|
||
| // Packed fp8 qkv buffer size for normal fp8 context FMHA | ||
|
|
@@ -858,8 +850,6 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL | |
| size_t const sage_k_mean_buffer_size = sage_attn_smooth_k ? sizeof(float) * local_hidden_units_kv : 0; | ||
| size_t const sage_k_mean_sum_buffer_size = sage_attn_smooth_k ? sizeof(int64_t) * local_hidden_units_kv : 0; | ||
|
|
||
| size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_length; | ||
| size_t const encoder_padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * cross_kv_length; | ||
| // Each token holds (batch_idx, token_idx_in_seq) int2. | ||
| size_t const tokens_info_size = sizeof(int2) * max_num_tokens; | ||
| size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0; | ||
|
|
@@ -868,24 +858,14 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL | |
|
|
||
| size_t const fmha_multi_ctas_kv_scratch_size = useTllmGenSparseAttention() ? getFmhaMultiCtasKvScratchSize(p) : 0; | ||
|
|
||
| AttentionContextWorkspaceSizes workspaceSizes{}; | ||
| workspaceSizes.attentionMask = attention_mask_size; | ||
| workspaceSizes.cuQSeqlens = cu_seqlens_size; | ||
| workspaceSizes.cuKvSeqlens = cu_seqlens_size; | ||
| workspaceSizes.cuMaskRows = cu_seqlens_size; | ||
| workspaceSizes.rotaryInvFreq = rotary_inv_freq_size; | ||
| workspaceSizes.qBuf = q_buf_2_size; | ||
| workspaceSizes.kBuf = k_buf_2_size; | ||
| workspaceSizes.vBuf = v_buf_2_size; | ||
| workspaceSizes.qkBuf = qk_buf_size; | ||
| workspaceSizes.qkvBuf = qkv_buf_2_size; | ||
| workspaceSizes.qkFloatBuf = qk_buf_float_size; | ||
| workspaceSizes.fp8QkvBuf = fp8_qkv_buffer_size; | ||
| workspaceSizes.fp8QBuf = fp8_q_buf_size; | ||
| workspaceSizes.fp8KBuf = fp8_k_buf_size; | ||
| workspaceSizes.fp8VBuf = fp8_v_buf_size; | ||
| workspaceSizes.paddingOffset = padding_offset_size; | ||
| workspaceSizes.encoderPaddingOffset = encoder_padding_offset_size; | ||
| workspaceSizes.tokensInfo = tokens_info_size; | ||
| workspaceSizes.fmhaTileCounter = fmha_scheduler_counter; | ||
| workspaceSizes.fmhaBmm1Scale = fmha_bmm1_scale_size; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Nice to have] (N1: Architecture and design)
buildUnfusedContextSizesis the only out-of-line member ofAttentionWorkspaceManager:buildContextLayout,materializeContextand the other builders are inline in attentionWorkspace.h (244-395). Here it gets a new STATIC archive, aTLLM_FACADE_COMPONENT_TARGETSentry and aPUBLIClink fromcommon_src(lines 126-128). None ofcommon_src's sources include attentionWorkspace.h, and the facade entry already whole-archives the library into libtensorrt_llm. Because${SHARED_TARGET}linkscommon_srcPUBLIC, the archive also lands on the link line of every libtensorrt_llm consumer. It also linkstllm::common_erroronly to get its include directories; the helper uses none of that library's symbols. Either define the function inline next to its siblings and link the test to a header-only target, or keep the component and drop thecommon_srcedge, depending ontllm::common_headersplus the CUDA include directories instead.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Narrowed attention workspace component dependencies.