Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
103dd9b
build: migrate CUTLASS MoE tests to explicit kernel components
Funatiq Oct 6, 2026
80bba31
[None][fix] link internal MoE component to FP8 block-scale GEMM
Funatiq Oct 24, 2026
d791323
[None][fix] bind CUTLASS cluster launches to their CUDA runtime
Funatiq Oct 31, 2026
bdb564f
refactor: rename CUDA core GEMM kernel directory
Funatiq Nov 16, 2026
5c8e907
build: extract custom all-reduce kernels into a focused component
Funatiq Oct 6, 2026
cb51469
build: migrate custom all-reduce tests through a runtime IPC component
Funatiq Oct 6, 2026
c99a22e
build: extract RMS-normalization kernels for component-local referenc…
Funatiq Oct 6, 2026
2adb7ac
build: migrate fused all-reduce tests to focused kernel components
Funatiq Oct 6, 2026
39b6d3b
build: migrate GEMM all-reduce tests to explicit kernel dependencies
Funatiq Oct 6, 2026
b1ebbd8
build: isolate NCCL resources from Torch tensor helpers
Funatiq Oct 6, 2026
ffaa920
build: migrate NCCL utility tests to focused communicator components
Funatiq Oct 6, 2026
520ef9e
fix: declare NCCL common component link dependencies
Funatiq Nov 16, 2026
f721197
fix: link NCCL communicator component to process-group utilities
Funatiq Nov 16, 2026
165ba9d
build: simplify explicit MoE GEMM source list
Funatiq Nov 20, 2026
79b3a26
build: remove stale custom all-reduce source filter
Funatiq Nov 20, 2026
d03292d
build: publish runtime IPC custom all-reduce dependency
Funatiq Nov 20, 2026
3c37e78
build: remove duplicate MoE test library dependency
Funatiq Nov 20, 2026
e039dbc
build: preserve kernel selection without changing ELF layout
Funatiq Nov 20, 2026
7af113c
fix: typo in the SPDX copyright notice
Funatiq Nov 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions cpp/cmake/templates/cutlass_runtime_symbols.map
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/* ClusterLauncher receives kernel entry points registered with this library's
* static CUDA runtime. Its inline helpers must not be interposed by another
* shared library's copies, which use a different runtime instance.
*/
{
local:
_ZN7cutlass15ClusterLauncher*;
};
13 changes: 13 additions & 0 deletions cpp/tensorrt_llm/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,16 @@ set_target_properties(
PROPERTIES CXX_STANDARD "20" CXX_STANDARD_REQUIRED "YES" CXX_EXTENSIONS "NO"
LINK_FLAGS "${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}")

if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
# Cluster launch helpers must use the CUDA runtime that registered the kernel.
set(CUTLASS_RUNTIME_SYMBOLS_MAP
"${PROJECT_SOURCE_DIR}/cmake/templates/cutlass_runtime_symbols.map")
target_link_options(${SHARED_TARGET} PRIVATE
"LINKER:--version-script,${CUTLASS_RUNTIME_SYMBOLS_MAP}")
set_property(TARGET ${SHARED_TARGET} APPEND
PROPERTY LINK_DEPENDS "${CUTLASS_RUNTIME_SYMBOLS_MAP}")
endif()

# Keep statically linked NVRTC symbols local so they cannot be interposed by a
# dynamic NVRTC loaded elsewhere in the process.
if(NOT WIN32 AND NOT NVRTC_DYNAMIC_LINKING)
Expand All @@ -323,6 +333,8 @@ set(TLLM_FACADE_COMPONENT_TARGETS
tllm_common_environment
tllm_common_error
tllm_common_logging
tllm_common_nccl
tllm_common_nccl_comm
tllm_common_profiler
tllm_context_transfer_consensus
tllm_executor_agent_name
Expand All @@ -340,6 +352,7 @@ set(TLLM_FACADE_COMPONENT_TARGETS
tllm_runtime_distributed
tllm_runtime_gdr
tllm_runtime_host_accessible_memory
tllm_runtime_ipc
tllm_runtime_locality
tllm_runtime_lora
tllm_runtime_lora_cache
Expand Down
42 changes: 32 additions & 10 deletions cpp/tensorrt_llm/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -92,15 +92,36 @@ if(ENABLE_CUBLASLT_FP4_GEMM)
target_compile_definitions(tllm_common_cublas PUBLIC ENABLE_CUBLASLT_FP4_GEMM)
endif()

add_library(tllm_common_nccl STATIC ncclUtils.cpp)
add_library(tllm::common_nccl ALIAS tllm_common_nccl)
target_link_libraries(tllm_common_nccl PUBLIC tllm::common_cuda)
target_compile_features(tllm_common_nccl PUBLIC cxx_std_20)
target_compile_definitions(
tllm_common_nccl PUBLIC ENABLE_MULTI_DEVICE=$<BOOL:${ENABLE_MULTI_DEVICE}>)
if(ENABLE_MULTI_DEVICE)
target_link_libraries(
tllm_common_nccl
PUBLIC ${NCCL_LIB} MPI::MPI_C torch_cpu c10
PRIVATE tllm::runtime_distributed)
endif()
set_target_properties(tllm_common_nccl PROPERTIES POSITION_INDEPENDENT_CODE ON)

add_library(tllm_common_nccl_comm STATIC ncclCommUtils.cpp)
add_library(tllm::common_nccl_comm ALIAS tllm_common_nccl_comm)
target_link_libraries(
tllm_common_nccl_comm
PUBLIC tllm::common_nccl
PRIVATE tllm::runtime_distributed tllm::runtime_ipc_nvls)
if(ENABLE_MULTI_DEVICE)
target_link_libraries(tllm_common_nccl_comm PRIVATE pg_utils)
endif()
target_compile_features(tllm_common_nccl_comm PUBLIC cxx_std_20)
set_target_properties(tllm_common_nccl_comm PROPERTIES POSITION_INDEPENDENT_CODE
ON)

add_library(
common_src OBJECT
cudaArchitectures.cpp
cudaFp8Utils.cu
mcastDevMemUtils.cpp
ncclUtils.cpp
opUtils.cpp
safetensors.cpp
sageQuant.cu)
common_src OBJECT cudaArchitectures.cpp cudaFp8Utils.cu mcastDevMemUtils.cpp
opUtils.cpp safetensors.cpp sageQuant.cu)

list(JOIN CMAKE_CUDA_ARCHITECTURES_ORIG "," TRTLLM_BUILT_CUDA_ARCHITECTURES)
list(JOIN CMAKE_CUDA_ARCHITECTURES_WITH_KERNELS ","
Expand All @@ -115,5 +136,6 @@ set_source_files_properties(
add_cuda_architectures(common_src 89)
set_property(TARGET common_src PROPERTY POSITION_INDEPENDENT_CODE ON)
set_property(TARGET common_src PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON)
target_link_libraries(common_src PUBLIC trtllm_gen_fmha_interface
tllm::common_cublas)
target_link_libraries(
common_src PUBLIC trtllm_gen_fmha_interface tllm::common_cublas
tllm::common_nccl_comm)
236 changes: 236 additions & 0 deletions cpp/tensorrt_llm/common/ncclCommUtils.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "tensorrt_llm/common/ncclUtils.h"
#include "tensorrt_llm/common/opUtils.h"
#include "tensorrt_llm/runtime/ipcNvlsMemory.h"
#include "tensorrt_llm/runtime/utils/mpiTags.h"
#include "tensorrt_llm/runtime/utils/mpiUtils.h"
#include "tensorrt_llm/runtime/utils/pgUtils.h"

#include <iterator>
#include <mutex>
#include <sstream>

TRTLLM_NAMESPACE_BEGIN

#if ENABLE_MULTI_DEVICE

std::unordered_map<tensorrt_llm::DataType, ncclDataType_t>* getDtypeMap()
{
static std::unordered_map<tensorrt_llm::DataType, ncclDataType_t> dtypeMap = {
{tensorrt_llm::DataType::kFLOAT, ncclFloat32},
{tensorrt_llm::DataType::kHALF, ncclFloat16},
{tensorrt_llm::DataType::kBF16, ncclBfloat16},
{tensorrt_llm::DataType::kFP8, ncclInt8},
{tensorrt_llm::DataType::kBOOL, ncclInt8},
{tensorrt_llm::DataType::kINT32, ncclInt32},
{tensorrt_llm::DataType::kINT64, ncclInt64},
{tensorrt_llm::DataType::kUINT8, ncclUint8},
{tensorrt_llm::DataType::kINT8, ncclInt8},
};
return &dtypeMap;
}

namespace
{

// Get NCCL unique ID for a group of ranks.
ncclUniqueId getUniqueId(std::set<int> const& group)
{
if (!useMPI())
{
auto const pg = pg_utils::get_world_pg();
TLLM_CHECK_WITH_INFO(pg != nullptr, "Host process group is not initialized");
pg_utils::PgHelper host(pg);
ncclUniqueId id{};
auto bytes = at::from_blob(&id, {sizeof(id)}, at::TensorOptions().dtype(at::kByte));
int const root = *group.begin();
int const tag = static_cast<int>(mpi::MpiTag::kNcclUniqueId);
if (pg->getRank() == root)
{
NCCLCHECK_THROW(ncclGetUniqueId(&id));
for (auto const rank : group)
{
if (rank != root)
{
PGCHECK_THROW(host.send(bytes, rank, tag));
}
}
}
else
{
PGCHECK_THROW(host.recv(bytes, root, tag));
}
return id;
}
auto const rank = COMM_SESSION.getRank();
TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, rank);
ncclUniqueId id;
if (rank == *group.begin())
{
NCCLCHECK_THROW(ncclGetUniqueId(&id));
for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it)
{
COMM_SESSION.sendValue(id, *it, tensorrt_llm::mpi::MpiTag::kNcclUniqueId);
}
}
else
{
COMM_SESSION.recvValue(id, *group.begin(), tensorrt_llm::mpi::MpiTag::kNcclUniqueId);
}
TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, rank);
return id;
}
} // namespace

std::shared_ptr<ncclComm_t> getComm(std::set<int> const& group)
{
auto const hostPg = useMPI() ? nullptr : pg_utils::get_world_pg();
TLLM_CHECK_WITH_INFO(useMPI() || hostPg != nullptr, "Host process group is not initialized");
auto const rank = useMPI() ? COMM_SESSION.getRank() : hostPg->getRank();
TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, rank);
static std::map<std::set<int>, std::shared_ptr<ncclComm_t>> commMap;
static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex);
std::ostringstream oss;
int index = 0;
for (auto const& rank : group)
{
if (index != 0)
{
oss << ",";
}
oss << rank;
index++;
}
auto groupStr = oss.str();
auto it = commMap.find(group);
if (it != commMap.end())
{
auto ncclComm = it->second;
TLLM_LOG_TRACE("NCCL comm for group(%s) is cached for rank %d", groupStr.c_str(), rank);
return ncclComm;
}

TLLM_LOG_TRACE("Init NCCL comm for group(%s) for rank %d", groupStr.c_str(), rank);
// Finish every environment update before getUniqueId(): on the group root, ncclGetUniqueId() starts NCCL's
// bootstrap thread, which reads the environment with getenv(), and a concurrent setenv() may reallocate
// environ underneath it.
#if defined(_WIN32)
// Need static connection initialization for accurate KV cache size estimation
if (getenv("NCCL_RUNTIME_CONNECT") == nullptr)
_putenv_s("NCCL_RUNTIME_CONNECT", "0");
// Disable graph register to avoid startup hangs
if (getenv("NCCL_GRAPH_REGISTER") == nullptr)
_putenv_s("NCCL_GRAPH_REGISTER", "0");
#else
setenv("NCCL_RUNTIME_CONNECT", "0", 0);
setenv("NCCL_GRAPH_REGISTER", "0", 0);
// NCCL aborts during init if it tries NVLS multicast but the fabric/IMEX
// plane can't bind it. Disable NVLS when the fabric is not usable so NCCL
// falls back to NVLink P2P. No-overwrite preserves an explicit user setting.
if (!tensorrt_llm::runtime::ipcNvlsFabricUsable())
{
setenv("NCCL_NVLS_ENABLE", "0", 0);
}
#endif // _WIN32
ncclUniqueId id = getUniqueId(group);
int groupRank = 0;
for (auto const& currentRank : group)
{
if (rank == currentRank)
break;
++groupRank;
}
TLLM_CHECK(static_cast<size_t>(groupRank) < group.size());
std::shared_ptr<ncclComm_t> ncclComm(new ncclComm_t,
[](ncclComm_t* comm)
{
if (!comm)
{
return;
}

// STEP 1: Clean up resources and destroy NCCL communicator if it's valid
if (*comm)
{
// Clean up all registered resources FIRST
// The cleanupResources function uses a destruction guard to safely handle
// static destruction order issues - it will return early if the singleton
// is being destroyed (in which case the destructor handles cleanup proactively)
tensorrt_llm::common::nccl_util::NcclCommResourceManager::getInstance().cleanupResources(*comm);

// Now destroy the NCCL communicator
ncclResult_t result = ncclCommDestroy(*comm);
if (result != ncclSuccess)
{
// Logging may fail during static destruction, so wrap in try-catch
try
{
TLLM_LOG_WARNING("ncclCommDestroy failed with error: %d", result);
}
catch (...)
{
// Ignore logging failures during static destruction
}
}

// Clear the communicator value before freeing the pointer
*comm = nullptr;
}

// STEP 2: Always free the pointer memory (regardless of whether *comm was valid)
delete comm;
});
#if NCCL_VERSION_CODE >= NCCL_VERSION(2, 29, 0)
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
config.graphUsageMode = 1;
NCCLCHECK_THROW(ncclCommInitRankConfig(ncclComm.get(), group.size(), id, groupRank, &config));
#else
NCCLCHECK_THROW(ncclCommInitRank(ncclComm.get(), group.size(), id, groupRank));
#endif // NCCL_VERSION_CODE >= NCCL_VERSION(2, 29, 0)
#if NCCL_VERSION_CODE >= NCCL_VERSION(2, 28, 0)
if (common::nccl_util::isNcclWindowSupported())
{
std::vector<int> const ranks(group.begin(), group.end());
if (useMPI())
{
common::nccl_util::registerHostCommunicator(*ncclComm, COMM_SESSION, ranks);
}
else
{
common::nccl_util::registerHostCommunicator(*ncclComm, hostPg, ranks);
}
}
#endif
commMap[group] = ncclComm;
TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, rank);
return ncclComm;
}
#endif // ENABLE_MULTI_DEVICE

void const* tensorrt_llm::common::op::getCommSessionHandle()
{
#if ENABLE_MULTI_DEVICE
return &COMM_SESSION;
#else
return nullptr;
#endif // ENABLE_MULTI_DEVICE
}

TRTLLM_NAMESPACE_END
Loading
Loading