Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions cpp/include/tensorrt_llm/common/cudaUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "tensorrt_llm/common/cudaBf16Wrapper.h"
#include "tensorrt_llm/common/cudaDriverWrapper.h"
#include "tensorrt_llm/common/cudaFp8Utils.h"
#include "tensorrt_llm/common/workspace.h"
#if ENABLE_FP4
#include <cuda_fp4.h>
#endif
Expand Down Expand Up @@ -55,9 +56,6 @@ TRTLLM_NAMESPACE_BEGIN
namespace common
{

// workspace for cublas gemm : 32MB
#define CUBLAS_WORKSPACE_SIZE 33554432

typedef struct __align__(4)
{
half x, y, z, w;
Expand Down
1 change: 1 addition & 0 deletions cpp/tensorrt_llm/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ set(TLLM_FACADE_COMPONENT_TARGETS
ipc_nvls_memory_src
tllm_batch_request_info
tllm_batch_scheduler
tllm_common_attention_workspace
tllm_common_cublas
tllm_common_cuda
tllm_common_environment
Expand Down
11 changes: 11 additions & 0 deletions cpp/tensorrt_llm/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,17 @@ if(ENABLE_CUBLASLT_FP4_GEMM)
target_compile_definitions(tllm_common_cublas PUBLIC ENABLE_CUBLASLT_FP4_GEMM)
endif()

add_library(tllm_common_attention_workspace STATIC attentionWorkspace.cpp)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Nice to have] (N1: Architecture and design) buildUnfusedContextSizes is the only out-of-line member of AttentionWorkspaceManager: buildContextLayout, materializeContext and the other builders are inline in attentionWorkspace.h (244-395). Here it gets a new STATIC archive, a TLLM_FACADE_COMPONENT_TARGETS entry and a PUBLIC link from common_src (lines 126-128). None of common_src's sources include attentionWorkspace.h, and the facade entry already whole-archives the library into libtensorrt_llm. Because ${SHARED_TARGET} links common_src PUBLIC, the archive also lands on the link line of every libtensorrt_llm consumer. It also links tllm::common_error only to get its include directories; the helper uses none of that library's symbols. Either define the function inline next to its siblings and link the test to a header-only target, or keep the component and drop the common_src edge, depending on tllm::common_headers plus the CUDA include directories instead.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Narrowed attention workspace component dependencies.

add_library(tllm::common_attention_workspace ALIAS
tllm_common_attention_workspace)
target_link_libraries(tllm_common_attention_workspace
PUBLIC tllm::common_headers)
target_include_directories(tllm_common_attention_workspace SYSTEM
PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
target_compile_features(tllm_common_attention_workspace PUBLIC cxx_std_20)
set_target_properties(tllm_common_attention_workspace
PROPERTIES POSITION_INDEPENDENT_CODE ON)

add_library(
common_src OBJECT
cudaArchitectures.cpp
Expand Down
51 changes: 51 additions & 0 deletions cpp/tensorrt_llm/common/attentionWorkspace.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "tensorrt_llm/common/attentionWorkspace.h"

TRTLLM_NAMESPACE_BEGIN

namespace common::op
{

AttentionContextWorkspaceSizes AttentionWorkspaceManager::buildUnfusedContextSizes(
AttentionUnfusedContextWorkspaceParams const& params) noexcept
{
AttentionContextWorkspaceSizes sizes{};
auto const paddedQueryTokens = params.batchSize * params.querySequenceLength;
auto const paddedKvTokens = params.batchSize * params.kvSequenceLength;
auto const queryHiddenUnits = params.numQueryHeads * params.headSize;
auto const kvHiddenUnits = params.numAttnKvHeads * params.headSize;
auto const scoreElements = paddedQueryTokens * params.numQueryHeads * params.kvSequenceLength;
sizes.attentionMask = params.elementSize * paddedQueryTokens * params.kvSequenceLength;
sizes.cuQSeqlens = sizeof(int) * (params.batchSize + 1);
sizes.cuKvSeqlens = sizes.cuQSeqlens;
sizes.cuMaskRows = sizes.cuQSeqlens;
sizes.qBuf = params.elementSize * paddedQueryTokens * queryHiddenUnits;
sizes.kBuf = params.elementSize * paddedKvTokens * kvHiddenUnits;
sizes.vBuf = sizes.kBuf;
sizes.qkBuf = params.elementSize * scoreElements;
sizes.qkvBuf = sizes.qBuf;
sizes.qkFloatBuf = sizeof(float) * scoreElements;
sizes.paddingOffset = sizeof(int) * paddedQueryTokens;
sizes.encoderPaddingOffset = sizeof(int) * params.batchSize * params.crossKvLength;
sizes.tokensInfo = sizeof(int2) * params.packedTokenCount;
return sizes;
}

} // namespace common::op

TRTLLM_NAMESPACE_END
24 changes: 23 additions & 1 deletion cpp/tensorrt_llm/common/attentionWorkspace.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
#pragma once

#include "tensorrt_llm/common/config.h"
#include "tensorrt_llm/common/cudaUtils.h"
#include "tensorrt_llm/common/workspace.h"

#include <cuda_fp8.h>
#include <vector_types.h>

#include <cstddef>
#include <cstdint>

Expand Down Expand Up @@ -70,6 +72,20 @@ struct AttentionContextWorkspaceSizes
size_t fmhaMultiCtasKvScratch{};
};

//! Geometry and element size for padded unfused context attention buffers.
struct AttentionUnfusedContextWorkspaceParams
{
size_t elementSize{};
size_t batchSize{};
size_t querySequenceLength{};
size_t kvSequenceLength{};
size_t crossKvLength{}; //!< Encoder sequence extent; zero for self-attention.
size_t packedTokenCount{};
int numQueryHeads{};
int numAttnKvHeads{};
int headSize{};
};

struct AttentionContextWorkspaceLayout
{
WorkspaceSlice cublasWorkspace{};
Expand Down Expand Up @@ -221,6 +237,12 @@ struct AttentionXqaWorkspaceLayout
class AttentionWorkspaceManager
{
public:
//! Size padded Q/K/V and score buffers, sequence metadata, and packed-token metadata.
//! Q/K/V and score buffers use padded counts because padding removal can make the packed token count smaller
//! than batch size times the maximum context query length.
[[nodiscard]] static AttentionContextWorkspaceSizes buildUnfusedContextSizes(
AttentionUnfusedContextWorkspaceParams const& params) noexcept;

static AttentionContextWorkspaceLayout buildContextLayout(
AttentionContextWorkspaceSizes const& sizes, uintptr_t alignment = common::kCudaMemAlign)
{
Expand Down
5 changes: 4 additions & 1 deletion cpp/tensorrt_llm/common/workspace.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 1993-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 1993-2026, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -18,6 +18,9 @@
#include <cstddef>
#include <cstdint>

// Workspace for cuBLAS GEMM: 32 MiB.
#define CUBLAS_WORKSPACE_SIZE 33554432

TRTLLM_NAMESPACE_BEGIN

namespace common
Expand Down
44 changes: 12 additions & 32 deletions cpp/tensorrt_llm/thop/attentionOp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -802,32 +802,24 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL

auto const batch_size = static_cast<size_t>(max_num_seq);
auto const kv_seq_length = (isCrossAttention(p) ? cross_kv_length : input_seq_length);
// The unfused-MHA buffers below must upper-bound the enqueueContext carve, which sizes them by
// batch_size * input_seq_length (not num_tokens): with padding removal the actual token count can be
// smaller than batch_size * max(context q length), so sizing by max_num_tokens underestimates.
size_t const attention_mask_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_length * kv_seq_length;
auto workspaceSizes = mEnableContextFMHA ? AttentionContextWorkspaceSizes{}
: AttentionWorkspaceManager::buildUnfusedContextSizes({.elementSize = size,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Nice to have] (R1: PR process) This hunk overlaps open PR #21915 ("Phased FMHA libraries, selected in Python"). At its head c64c037, MHA::contextWorkspaceSizes (thop/attentionOp.cpp:320) sizes only the fused buffers. A new UnfusedFmha::contextWorkspaceLayout (thop/phasedFmha.cpp:2525-2557) re-implements the attention mask, Q/K/V, score and padding-offset sizes inline, with the old "must upper-bound the carve" comment. If #21915 lands first, this change has to be redone there. If this PR lands first and #21915 keeps its inline copy when it rebases, buildUnfusedContextSizes has no production caller left, and the new host tests check sizing that production no longer uses. Could you link the two PRs and agree the order with the #21915 author, ideally so that UnfusedFmha::contextWorkspaceLayout calls this helper?

.batchSize = batch_size,
.querySequenceLength = static_cast<size_t>(input_seq_length),
.kvSequenceLength = static_cast<size_t>(kv_seq_length),
.crossKvLength = static_cast<size_t>(cross_kv_length),
.packedTokenCount = static_cast<size_t>(max_num_tokens),
.numQueryHeads = mNumAttnHeads,
.numAttnKvHeads = mNumAttnKVHeads,
.headSize = getHeadSize()});
size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1);
size_t const rotary_inv_freq_size = sizeof(float) * batch_size * rotary_embedding_dim / 2;

size_t q_buf_2_size = 0;
if (!mEnableContextFMHA)
{
// Unfused mha
q_buf_2_size = size * batch_size * input_seq_length * local_hidden_units_qo;
}
else if (mFmhaDispatcher->isSeparateQAndKvInput())
if (mEnableContextFMHA && mFmhaDispatcher->isSeparateQAndKvInput())
{
// Paged context fmha
q_buf_2_size = (mFP8ContextFMHA ? 1 : size) * max_num_tokens * local_hidden_units_qo;
workspaceSizes.qBuf = (mFP8ContextFMHA ? 1 : size) * max_num_tokens * local_hidden_units_qo;
}

size_t const k_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * kv_seq_length * local_hidden_units_kv;
size_t const v_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * kv_seq_length * local_hidden_units_kv;
size_t const qk_buf_size
= mEnableContextFMHA ? 0 : size * batch_size * num_heads * input_seq_length * kv_seq_length;
size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_length * local_hidden_units_qo;
size_t const qk_buf_float_size
= mEnableContextFMHA ? 0 : sizeof(float) * batch_size * num_heads * input_seq_length * kv_seq_length;
bool const useSageAttnSeparateQkv = mEnableContextFMHA && useSageAttn() && mFmhaDispatcher->isSeparateQAndKvInput();

// Packed fp8 qkv buffer size for normal fp8 context FMHA
Expand Down Expand Up @@ -858,8 +850,6 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL
size_t const sage_k_mean_buffer_size = sage_attn_smooth_k ? sizeof(float) * local_hidden_units_kv : 0;
size_t const sage_k_mean_sum_buffer_size = sage_attn_smooth_k ? sizeof(int64_t) * local_hidden_units_kv : 0;

size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_length;
size_t const encoder_padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * cross_kv_length;
// Each token holds (batch_idx, token_idx_in_seq) int2.
size_t const tokens_info_size = sizeof(int2) * max_num_tokens;
size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0;
Expand All @@ -868,24 +858,14 @@ tensorrt_llm::common::op::AttentionContextWorkspaceLayout MHA::contextWorkspaceL

size_t const fmha_multi_ctas_kv_scratch_size = useTllmGenSparseAttention() ? getFmhaMultiCtasKvScratchSize(p) : 0;

AttentionContextWorkspaceSizes workspaceSizes{};
workspaceSizes.attentionMask = attention_mask_size;
workspaceSizes.cuQSeqlens = cu_seqlens_size;
workspaceSizes.cuKvSeqlens = cu_seqlens_size;
workspaceSizes.cuMaskRows = cu_seqlens_size;
workspaceSizes.rotaryInvFreq = rotary_inv_freq_size;
workspaceSizes.qBuf = q_buf_2_size;
workspaceSizes.kBuf = k_buf_2_size;
workspaceSizes.vBuf = v_buf_2_size;
workspaceSizes.qkBuf = qk_buf_size;
workspaceSizes.qkvBuf = qkv_buf_2_size;
workspaceSizes.qkFloatBuf = qk_buf_float_size;
workspaceSizes.fp8QkvBuf = fp8_qkv_buffer_size;
workspaceSizes.fp8QBuf = fp8_q_buf_size;
workspaceSizes.fp8KBuf = fp8_k_buf_size;
workspaceSizes.fp8VBuf = fp8_v_buf_size;
workspaceSizes.paddingOffset = padding_offset_size;
workspaceSizes.encoderPaddingOffset = encoder_padding_offset_size;
workspaceSizes.tokensInfo = tokens_info_size;
workspaceSizes.fmhaTileCounter = fmha_scheduler_counter;
workspaceSizes.fmhaBmm1Scale = fmha_bmm1_scale_size;
Expand Down
4 changes: 2 additions & 2 deletions cpp/tests/unit_tests/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ add_tllm_gtest(
SOURCES
attentionWorkspaceTest.cpp
LINK_LIBRARIES
tllm::common_logging
tllm::common_attention_workspace
LABELS
"unit;common;host")
"unit;common;attention;host")
add_tllm_gtest(
memoryUtilsTest
OWNER
Expand Down
86 changes: 86 additions & 0 deletions cpp/tests/unit_tests/common/attentionWorkspaceTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,92 @@ void expectNextSlice(char const* name, Slice const& slice, size_t size, size_t&

} // namespace

TEST(AttentionWorkspaceManagerTest, RaggedUnfusedSelfAttentionUsesPaddedTokenCounts)
{
auto const sizes = tcop::AttentionWorkspaceManager::buildUnfusedContextSizes({.elementSize = sizeof(half),
.batchSize = 4,
.querySequenceLength = 64,
.kvSequenceLength = 64,
.crossKvLength = 0,
.packedTokenCount = 100,
.numQueryHeads = 8,
.numAttnKvHeads = 2,
.headSize = 64});

EXPECT_EQ(sizes.attentionMask, 32768);
EXPECT_EQ(sizes.cuQSeqlens, 20);
EXPECT_EQ(sizes.cuKvSeqlens, 20);
EXPECT_EQ(sizes.cuMaskRows, 20);
EXPECT_EQ(sizes.qBuf, 262144);
EXPECT_EQ(sizes.kBuf, 65536);
EXPECT_EQ(sizes.vBuf, 65536);
EXPECT_EQ(sizes.qkBuf, 262144);
EXPECT_EQ(sizes.qkvBuf, 262144);
EXPECT_EQ(sizes.qkFloatBuf, 524288);
EXPECT_EQ(sizes.paddingOffset, 1024);
EXPECT_EQ(sizes.encoderPaddingOffset, 0);
EXPECT_EQ(sizes.tokensInfo, 800);
}

TEST(AttentionWorkspaceManagerTest, RaggedUnfusedCrossAttentionUsesPaddedTokenCounts)
{
auto const sizes = tcop::AttentionWorkspaceManager::buildUnfusedContextSizes({.elementSize = sizeof(half),
.batchSize = 4,
.querySequenceLength = 64,
.kvSequenceLength = 32,
.crossKvLength = 32,
.packedTokenCount = 100,
.numQueryHeads = 8,
.numAttnKvHeads = 2,
.headSize = 64});

EXPECT_EQ(sizes.attentionMask, 16384);
EXPECT_EQ(sizes.cuQSeqlens, 20);
EXPECT_EQ(sizes.cuKvSeqlens, 20);
EXPECT_EQ(sizes.cuMaskRows, 20);
EXPECT_EQ(sizes.qBuf, 262144);
EXPECT_EQ(sizes.kBuf, 32768);
EXPECT_EQ(sizes.vBuf, 32768);
EXPECT_EQ(sizes.qkBuf, 131072);
EXPECT_EQ(sizes.qkvBuf, 262144);
EXPECT_EQ(sizes.qkFloatBuf, 262144);
EXPECT_EQ(sizes.paddingOffset, 1024);
EXPECT_EQ(sizes.encoderPaddingOffset, 512);
EXPECT_EQ(sizes.tokensInfo, 800);
}

TEST(AttentionWorkspaceManagerTest, UnfusedGroupedQuerySizesDistinguishQueryAndKvHeads)
{
tcop::AttentionUnfusedContextWorkspaceParams params{.elementSize = sizeof(half),
.batchSize = 3,
.querySequenceLength = 5,
.kvSequenceLength = 2,
.crossKvLength = 2,
.packedTokenCount = 7,
.numQueryHeads = 4,
.numAttnKvHeads = 1,
.headSize = 8};
auto const sizes = tcop::AttentionWorkspaceManager::buildUnfusedContextSizes(params);
EXPECT_EQ(sizes.attentionMask, 60);
EXPECT_EQ(sizes.qBuf, 960);
EXPECT_EQ(sizes.qkvBuf, 960);
EXPECT_EQ(sizes.kBuf, 96);
EXPECT_EQ(sizes.vBuf, 96);
EXPECT_EQ(sizes.qkBuf, 240);
EXPECT_EQ(sizes.qkFloatBuf, 480);
EXPECT_EQ(sizes.paddingOffset, 60);
EXPECT_EQ(sizes.encoderPaddingOffset, 24);
EXPECT_EQ(sizes.tokensInfo, sizeof(int2) * 7);

params.packedTokenCount = 15;
auto const denseSizes = tcop::AttentionWorkspaceManager::buildUnfusedContextSizes(params);
EXPECT_EQ(denseSizes.qBuf, sizes.qBuf);
EXPECT_EQ(denseSizes.kBuf, sizes.kBuf);
EXPECT_EQ(denseSizes.qkBuf, sizes.qkBuf);
EXPECT_EQ(denseSizes.paddingOffset, sizes.paddingOffset);
EXPECT_EQ(denseSizes.tokensInfo, sizeof(int2) * 15);
}

TEST(AttentionWorkspaceManagerTest, ContextLayoutMatchesAttentionOpOrdering)
{
tcop::AttentionContextWorkspaceSizes sizes{};
Expand Down
Loading