From 33ac8e3770bf8cacb7986173c6887eafef777c1d Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Wed, 18 Nov 2026 01:39:24 -0800 Subject: [PATCH 01/13] [None][feat] Wide-EP fault tolerance: enable_wide_ep_fault_tolerance and a fault-tolerant MoE all-to-all PR 1 of 7 of Wide-EP fault tolerance: a wide expert-parallel deployment that keeps serving on fewer GPUs when a rank fails, and takes the rank back when its replacement starts. - Add the prototype field TorchLlmArgs.enable_wide_ep_fault_tolerance (default off). - With the field on, the NVLink one-sided all-to-all reads its rank mask from GPU memory, so captured CUDA graphs follow a membership change without recapture. - The host can abort the all-to-all waits on a peer; a long wait records which peer it waits on; no rank reads the rows of a round that a failure cut short. - Remove main's earlier fault-tolerant MoE all-to-all (the host-side AlltoAll watchdog and wide_ep_ft.py, selected by TLLM_FAULT_TOLERANCE_MODE=1) with its knobs. - With the field off, the all-to-all runs main's kernels and main's host path. Signed-off-by: Andrew Rusli --- .../moe/communication/moeAlltoAllKernels.cu | 503 +++++++++-- .../moe/communication/moeAlltoAllKernels.h | 102 ++- .../thop/moe/communication/moeAlltoAllMeta.h | 9 +- .../thop/moe/communication/moeAlltoAllOp.cpp | 143 +++- docs/source/developer-guide/moe-perf-guide.md | 2 +- tensorrt_llm/_torch/alltoall_watchdog.py | 792 ------------------ .../_torch/custom_ops/cpp_custom_ops.py | 12 + .../_torch/mnnvl_alltoall_workspace.py | 139 +-- tensorrt_llm/_torch/model_config.py | 2 + .../moe/fused_moe/MOE_DEVELOPER_GUIDE.md | 6 +- .../communication/communication_factory.py | 20 +- .../communication/execution_state.py | 158 ++++ .../communication/nvlink_one_sided.py | 184 ++-- .../_torch/moe/fused_moe/ep_group_health.py | 1 - .../_torch/moe/fused_moe/wide_ep_ft.py | 94 --- .../_torch/pyexecutor/error_classification.py | 1 - .../_torch/pyexecutor/model_loader.py | 7 + .../pyexecutor/rank_failure_model_load.py | 75 ++ tensorrt_llm/llmapi/llm_args.py | 8 + .../schemas/trtllm-serve-config.schema.json | 7 + .../trtllm-serve-disagg-config.schema.json | 14 + .../usage/llm_args_golden_manifest.json | 5 + .../test_lists/test-db/l0_b200.yml | 2 + .../test-db/l0_gb200_multi_gpus.yml | 4 + .../_torch/modules/test_alltoall_watchdog.py | 669 --------------- .../unittest/_torch/moe/moe_a2a_test_utils.py | 47 ++ .../moe/multi_gpu/test_nvlink_one_sided.py | 281 +++++-- .../_torch/moe/test_communication_factory.py | 57 ++ .../unittest/_torch/moe/test_moe_a2a_abort.py | 331 ++++++++ .../_torch/moe/test_moe_a2a_fake_ops.py | 58 ++ tests/unittest/_torch/moe/test_moe_comm.py | 24 +- .../_torch/moe/test_moe_execution_state.py | 370 ++++++++ tests/unittest/_torch/moe/test_moe_impl.py | 2 +- .../_torch/moe/test_nvlink_one_sided_host.py | 58 ++ .../pyexecutor/test_error_classification.py | 1 - .../_torch/test_mnnvl_alltoall_workspace.py | 216 +---- .../api_stability/references/llm.yaml | 4 + tests/unittest/llmapi/test_llm_args.py | 8 + .../test_trtllm_serve_config_schemas.py | 2 + 39 files changed, 2222 insertions(+), 2196 deletions(-) delete mode 100644 tensorrt_llm/_torch/alltoall_watchdog.py create mode 100644 tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py delete mode 100644 tensorrt_llm/_torch/moe/fused_moe/wide_ep_ft.py create mode 100644 tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py delete mode 100644 tests/unittest/_torch/modules/test_alltoall_watchdog.py create mode 100644 tests/unittest/_torch/moe/moe_a2a_test_utils.py create mode 100644 tests/unittest/_torch/moe/test_moe_a2a_abort.py create mode 100644 tests/unittest/_torch/moe/test_moe_a2a_fake_ops.py create mode 100644 tests/unittest/_torch/moe/test_moe_execution_state.py diff --git a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu index 97e398f0a6fb..d8b51527e329 100644 --- a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu @@ -210,12 +210,74 @@ __device__ __forceinline__ int compute_target_rank_id(int expert_id, int base, i return remainder + (expert_id - split) / base; } -// Test bit `rank` in a kRankMaskWords-wide little-endian uint64 bitmask. -// Word 0 covers ranks 0..63, word 1 covers ranks 64..127, etc. -// `rank >> 6` and `rank & 63` divide / modulo by 64. -__device__ __forceinline__ bool is_rank_active(uint64_t const* mask, int rank) +//! Copy the active-rank bitmask into registers. The local rank's bit is always set, because a rank never excludes +//! itself. +__device__ __forceinline__ void load_rank_mask_words(uint64_t const* mask, int rank, uint64_t (&words)[kRankMaskWords]) { - return (mask[rank >> 6] >> (rank & 63)) & 1ULL; +#pragma unroll + for (int w = 0; w < kRankMaskWords; ++w) + { + words[w] = mask[w] | ((rank >> 6) == w ? 1ULL << (rank & 63) : 0ULL); + } +} + +//! Test bit `rank` in a register copy of the mask without dynamic indexing, so that `words` stays in registers. +__device__ __forceinline__ bool is_rank_active_in_words(uint64_t const (&words)[kRankMaskWords], int rank) +{ + uint64_t word = words[0]; +#pragma unroll + for (int w = 1; w < kRankMaskWords; ++w) + { + word = (rank >> 6) == w ? words[w] : word; + } + return (word >> (rank & 63)) & 1ULL; +} + +//! Clear every rank except `rank` in a register copy of the mask. +__device__ __forceinline__ void keep_only_rank_in_words(uint64_t (&words)[kRankMaskWords], int rank) +{ +#pragma unroll + for (int w = 0; w < kRankMaskWords; ++w) + { + words[w] = (rank >> 6) == w ? words[w] & (1ULL << (rank & 63)) : 0; + } +} + +//! Return the incomplete-round value of a round flag value (see the incomplete-round word in the header). +__device__ __forceinline__ uint32_t incomplete_round_value(uint32_t flag_value) +{ + return flag_value | 1U; +} + +//! Return whether only this rank's own rows are valid in the round of `flag_value`, its dispatch or combine flag value. +__device__ __forceinline__ bool only_own_rows_valid(uint32_t incomplete_round, uint32_t flag_value) +{ + uint32_t const combine_flag_value = flag_value + (flag_value & 1U); + return incomplete_round != 0 && incomplete_round != incomplete_round_value(combine_flag_value); +} + +//! Return the incomplete-round word of `rank`. Requires ptrs.incomplete_round_offset != 0. +__device__ __forceinline__ uint32_t* incomplete_round_word(DispatchKernelPointers const& ptrs, int rank) +{ + return reinterpret_cast( + reinterpret_cast(ptrs.completion_flags[rank]) + ptrs.incomplete_round_offset); +} + +//! Store `value` in the incomplete-round word of every rank in `rank_mask` whose word is 0, then fence.sc.sys before +//! any later write of this rank to rows that the peers may read. The fence pairs with the fence.sc.sys of the +//! incomplete-round check that each peer runs after its forward (moeA2ACheckIncompleteRoundKernel). +__device__ __forceinline__ void mark_round_incomplete( + DispatchKernelPointers const& ptrs, uint64_t const (&rank_mask)[kRankMaskWords], int ep_size, uint32_t value) +{ +#pragma unroll 1 + for (int rank = 0; rank < ep_size; ++rank) + { + if (is_rank_active_in_words(rank_mask, rank)) + { + atomicCAS_system(incomplete_round_word(ptrs, rank), 0U, value); + } + } + asm volatile("fence.sc.sys;" ::: "memory"); } // Each A2A round uses two consecutive flag values: 2N + 1 for dispatch and @@ -233,8 +295,88 @@ __device__ __forceinline__ void publish_round_flag(uint32_t* address, uint32_t v asm volatile("st.relaxed.sys.u32 [%0], %1;" ::"l"(address), "r"(value) : "memory"); } -__device__ __forceinline__ bool wait_round_flag( - uint32_t const* address, uint32_t expected, int64_t timeout_cycles, int rank_id, int peer_rank, char const* phase) +//! Record in the abort state that this kernel returned without the data of some active source. +__device__ __forceinline__ void record_cancelled(MoeA2AAbortState* abort) +{ + asm volatile("st.relaxed.sys.u32 [%0], %1;" ::"l"(&abort->status), "r"(MoeA2AAbortState::kCancelled)); +} + +//! Return the value that a wait for round flag value `flag_value` publishes: the flag value, or 1 for a combine flag +//! value that wrapped to 0, since a published 0 means no wait. The wrapped combine then shares the value of the next +//! dispatch. That is harmless, because the host only groups waits on a peer by this value. +__device__ __forceinline__ uint32_t waited_flag_value(uint32_t flag_value) +{ + return flag_value != 0 ? flag_value : 1U; +} + +//! Publish the flag value that this rank waits for from `peer_rank` (value != 0), or withdraw it (value == 0). +__device__ __forceinline__ void publish_waited_flag(MoeA2AAbortState* abort, int peer_rank, uint32_t value) +{ + asm volatile("st.relaxed.sys.u32 [%0], %1;" ::"l"(&abort->waited_flag[peer_rank]), "r"(value) : "memory"); +} + +//! Abort polling state of a block. The abort state is in host memory, so to keep the reads off the host link one +//! waiting thread per kAbortPollCycles reads it: the thread whose atomicCAS moves `next_poll` past the current clock. +//! `requested` caches a seen request for the other threads of the block. +struct AbortPoll +{ + unsigned long long next_poll; + int requested; +}; + +//! Set up the block's AbortPoll, or return nullptr without ENABLE_RANK_MASK. +//! Call from one warp per block only. +template +__device__ __forceinline__ AbortPoll* start_abort_poll(int lane_id) +{ + if constexpr (ENABLE_RANK_MASK) + { + __shared__ AbortPoll poll; + if (lane_id == 0) + { + poll.next_poll = 0; + poll.requested = 0; + } + __syncwarp(); + return &poll; + } + else + { + return nullptr; + } +} + +//! Return whether the host requested an abort, and mark this kernel cancelled when it did. clock64() is per SM, so +//! all threads of the block compare one clock. +__device__ __forceinline__ bool abort_requested(MoeA2AAbortState* abort, AbortPoll* poll) +{ + if (*reinterpret_cast(&poll->requested) != 0) + { + return true; + } + auto const now = static_cast(clock64()); + auto const due = *reinterpret_cast(&poll->next_poll); + if (now < due || atomicCAS(&poll->next_poll, due, now + kAbortPollCycles) != due) + { + return false; + } + int32_t requested; + asm volatile("ld.relaxed.sys.u32 %0, [%1];" : "=r"(requested) : "l"(&abort->requested)); + if (requested == 0) + { + return false; + } + record_cancelled(abort); + *reinterpret_cast(&poll->requested) = 1; + return true; +} + +//! Wait until a peer's round flag reaches `expected`, and trap on timeout. With HONOR_ABORT, return false on abort. +//! The first loop is the wait without HONOR_ABORT. With HONOR_ABORT it ends after kAbortPollCycles, and the second +//! loop publishes the waited flag after kPublishWaitedFlagCycles and polls the abort request. +template +__device__ __forceinline__ bool wait_round_flag(uint32_t const* address, uint32_t expected, int64_t timeout_cycles, + MoeA2AAbortState* abort, AbortPoll* poll, int rank_id, int peer_rank, char const* phase) { auto const start = clock64(); uint32_t observed; @@ -249,7 +391,34 @@ __device__ __forceinline__ bool wait_round_flag( { return true; } - } while (!check_timeout(start, timeout_cycles)); + } while (HONOR_ABORT ? clock64() - start < kAbortPollCycles : !check_timeout(start, timeout_cycles)); + + if constexpr (HONOR_ABORT) + { + bool published = false; + bool aborted = false; + do + { + asm volatile("ld.relaxed.sys.u32 %0, [%1];" : "=r"(observed) : "l"(address) : "memory"); + if (observed != expected && abort != nullptr) + { + if (!published && clock64() - start >= kPublishWaitedFlagCycles) + { + publish_waited_flag(abort, peer_rank, waited_flag_value(expected)); + published = true; + } + aborted = abort_requested(abort, poll); + } + if (observed == expected || aborted) + { + if (published) + { + publish_waited_flag(abort, peer_rank, 0); + } + return !aborted; + } + } while (!check_timeout(start, timeout_cycles)); + } printf("%s: rank %d timed out waiting for rank %d flag=%u expected=%u\n", phase, rank_id, peer_rank, observed, expected); @@ -265,10 +434,11 @@ __host__ __device__ constexpr uint32_t make_warp_lane_mask() return (NUM_LANES == 32) ? ~0U : ((1U << NUM_LANES) - 1U); } +//! Route one token to its target ranks, skipping inactive ranks with ENABLE_RANK_MASK. template __device__ __forceinline__ void route_dispatch_token(int32_t const* token_selected_experts, - DispatchKernelPointers const& ptrs, int local_token_idx, int ep_size, int num_experts, int* topk_target_ranks, - int* topk_target_indices, uint32_t lane_mask) + DispatchKernelPointers const& ptrs, uint64_t const (&rank_mask)[kRankMaskWords], int local_token_idx, int ep_size, + int num_experts, int* topk_target_ranks, int* topk_target_indices, uint32_t lane_mask) { static_assert(TOP_K <= 32, "warp-parallel routing requires TOP_K <= warpSize"); int const k = threadIdx.x; @@ -293,7 +463,7 @@ __device__ __forceinline__ void route_dispatch_token(int32_t const* token_select bool should_send = valid_expert && ((__ffs(same_target) - 1) == k); if constexpr (ENABLE_RANK_MASK) { - should_send = should_send && is_rank_active(ptrs.active_rank_mask, target_rank); + should_send = should_send && is_rank_active_in_words(rank_mask, target_rank); } int const target_rank_to_store = should_send ? target_rank : -1; @@ -457,6 +627,28 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ int thread_idx = threadIdx.x; int local_token_idx = blockIdx.x; + // Load after the grid dependency wait, so that the writes of the mask and of the incomplete-round word by earlier + // work are visible. A peer may set the word while this kernel runs, so blocks may load different values. The + // block that finishes its token last (is_last_token) synchronizes with the peers according to its own load. + uint32_t incomplete = 0; + uint64_t rank_mask[kRankMaskWords] = {}; + auto const load_rank_mask = [&]() + { + if constexpr (ENABLE_RANK_MASK) + { + if (thread_idx < warpSize) + { + load_rank_mask_words(ptrs.active_rank_mask, rank_id, rank_mask); + incomplete = ptrs.incomplete_round_offset != 0 ? *incomplete_round_word(ptrs, rank_id) : 0; + // Active peers may still read the rows of an incomplete collective, so exchange with this rank only. + if (incomplete != 0) + { + keep_only_rank_in_words(rank_mask, rank_id); + } + } + } + }; + if (local_num_tokens == 0) { // Special case: If local_num_tokens == 0, @@ -467,6 +659,7 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ #if TLLM_MOE_A2A_COMPILE_SM90 cudaGridDependencySynchronize(); #endif + load_rank_mask(); } else { @@ -483,13 +676,14 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ #if TLLM_MOE_A2A_COMPILE_SM90 cudaGridDependencySynchronize(); #endif + load_rank_mask(); static_assert(TOP_K > 0 && TOP_K <= 32, "routing and destination setup require one warp"); constexpr uint32_t kRoutingLaneMask = make_warp_lane_mask(); __shared__ uint8_t* destinations[kMaxPayloads][TOP_K]; if (thread_idx < TOP_K) { route_dispatch_token(token_selected_experts, ptrs, - local_token_idx, ep_size, num_experts, smem_topk_target_ranks, smem_topk_target_indices, + rank_mask, local_token_idx, ep_size, num_experts, smem_topk_target_ranks, smem_topk_target_indices, kRoutingLaneMask); // Compact routing slots may be written by a different lane. __syncwarp(kRoutingLaneMask); @@ -550,6 +744,14 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ if (is_last_token) { + // The MoE after this dispatch may overwrite rows that active peers still read, so their combine must + // discard what it read. + if (ENABLE_RANK_MASK && incomplete != 0 && lane_id == 0) + { + uint64_t active_ranks[kRankMaskWords]; + load_rank_mask_words(ptrs.active_rank_mask, rank_id, active_ranks); + mark_round_incomplete(ptrs, active_ranks, ep_size, incomplete); + } uint32_t const parity = round_parity(*ptrs.flag_val); int* const round_recv_counters = ptrs.recv_counters[rank_id] + parity * ep_size; // Store send_counters to recv_counters. @@ -559,7 +761,7 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, target_rank)) + if (!is_rank_active_in_words(rank_mask, target_rank)) continue; } int send_count = ptrs.send_counters[target_rank]; @@ -575,7 +777,7 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, target_rank)) + if (!is_rank_active_in_words(rank_mask, target_rank)) continue; } int* target_stats = ptrs.eplb_gathered_stats[target_rank]; @@ -603,7 +805,7 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, target_rank)) + if (!is_rank_active_in_words(rank_mask, target_rank)) continue; } uint32_t* flag_addr = &ptrs.completion_flags[target_rank][rank_id]; @@ -615,6 +817,8 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ #endif } + AbortPoll* abort_poll = start_abort_poll(lane_id); + // Wait for all active peers to signal; skip dead ranks (otherwise we would // spin forever — this is the bug the rank-mask is here to prevent). #pragma unroll 1 // No unroll @@ -622,15 +826,29 @@ __global__ void moeA2ADispatchKernel(int32_t const* token_selected_experts, // [ { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, peer_rank)) + if (!is_rank_active_in_words(rank_mask, peer_rank)) { round_recv_counters[peer_rank] = 0; continue; } } - if (!wait_round_flag(&ptrs.completion_flags[rank_id][peer_rank], expected_value, ptrs.timeout_cycles, - rank_id, peer_rank, "dispatch")) + if (!wait_round_flag(&ptrs.completion_flags[rank_id][peer_rank], expected_value, + ptrs.timeout_cycles, ptrs.abort, abort_poll, rank_id, peer_rank, "dispatch")) { + if constexpr (ENABLE_RANK_MASK) + { + // Count no tokens from the peers this lane did not see complete, as from an inactive peer: + // if a peer already set the word to this round's combine value, the later kernels of this + // round go by these counts. This lane stored this rank's own count above. + for (int unconfirmed = peer_rank; unconfirmed < ep_size; unconfirmed += warpSize) + { + if (unconfirmed != rank_id) + { + round_recv_counters[unconfirmed] = 0; + } + } + mark_round_incomplete(ptrs, rank_mask, ep_size, incomplete_round_value(expected_value)); + } return; } } @@ -660,12 +878,16 @@ void moe_a2a_dispatch_launch(MoeA2ADispatchParams const& params) TLLM_CHECK(params.ep_rank >= 0 && params.ep_rank < params.ep_size); TLLM_CHECK(params.local_num_tokens >= 0); TLLM_CHECK(params.num_payloads > 0 && params.num_payloads <= kMaxPayloads); - // The local rank must always be marked active in its own view of the mask; - // otherwise the kernel itself would be running on a "dead" rank. if (params.enable_rank_mask) { - TLLM_CHECK_WITH_INFO((params.active_rank_mask[params.ep_rank >> 6] >> (params.ep_rank & 63)) & 1ULL, - "active_rank_mask must mark the local ep_rank (%d) as active", params.ep_rank); + TLLM_CHECK_WITH_INFO( + params.active_rank_mask != nullptr, "rank-mask mode needs the device address of active_rank_mask"); + } + if (params.abort != nullptr) + { + TLLM_CHECK_WITH_INFO( + params.incomplete_round_offset != 0, "an abort state needs a workspace laid out with rank_mask"); + TLLM_CHECK_WITH_INFO(params.enable_rank_mask, "the abort state needs enable_rank_mask"); } // Prepare kernel pointers struct @@ -697,6 +919,7 @@ void moe_a2a_dispatch_launch(MoeA2ADispatchParams const& params) kernel_ptrs.completion_flags[i] = params.completion_flags[i]; } kernel_ptrs.flag_val = params.flag_val; + kernel_ptrs.incomplete_round_offset = params.incomplete_round_offset; // Copy communication tracking pointers kernel_ptrs.send_counters = params.send_counters; @@ -705,11 +928,8 @@ void moe_a2a_dispatch_launch(MoeA2ADispatchParams const& params) kernel_ptrs.topk_target_indices = params.topk_target_indices; kernel_ptrs.eplb_local_stats = params.eplb_local_stats; - // Copy active-rank bitmask into the kernel pointers struct - for (int w = 0; w < kRankMaskWords; ++w) - { - kernel_ptrs.active_rank_mask[w] = params.active_rank_mask[w]; - } + kernel_ptrs.active_rank_mask = params.active_rank_mask; + kernel_ptrs.abort = params.abort; int grid_size = params.local_num_tokens; if (grid_size == 0) @@ -862,9 +1082,11 @@ __device__ void vectorized_combine_impl( // Pack valid source pointers in routing order and count the contributing ranks. // stride_per_token can exceed the wire size for in-place low-precision combine. -template +// With ENABLE_RANK_MASK and only_rank >= 0, only the routes whose dispatch target is only_rank contribute, so the +// combine reads only the rows that only_rank wrote. +template __device__ void vectorized_combine( - OutputT* output, int size_per_token, int stride_per_token, CombineKernelPointers const& ptrs) + OutputT* output, int size_per_token, int stride_per_token, CombineKernelPointers const& ptrs, int only_rank = -1) { static_assert(TOP_K > 0 && TOP_K <= 32, "combine routing requires TOP_K <= warpSize"); constexpr uint32_t kRoutingLaneMask = make_warp_lane_mask(); @@ -874,7 +1096,14 @@ __device__ void vectorized_combine( { int const k = threadIdx.x; int const index = blockIdx.x * TOP_K + k; - int const target_index = ptrs.topk_target_indices[index]; + int target_index = ptrs.topk_target_indices[index]; + if constexpr (ENABLE_RANK_MASK) + { + if (only_rank >= 0 && ptrs.topk_target_ranks[index] != only_rank) + { + target_index = -1; + } + } uint32_t const valid = __ballot_sync(kRoutingLaneMask, target_index >= 0); if (k == 0) { @@ -1003,17 +1232,22 @@ __device__ void vectorized_quant(DstT* dst, SrcT const* src, int num_elements) // Advance flag_val to the combine phase and prepare valid tokens in the requested range. // Copy SrcT payloads, or quantize them to FP8 when LOW_PRECISION is enabled, into combine_input_base. // This kernel performs no remote transfers or reduction. -template -__global__ void moeA2APrepareCombineKernel(uint8_t* combine_input_base, void const* source_payload, +// With HAS_INCOMPLETE_ROUND, only this rank's own tokens are valid while only_own_rows_valid() holds for the +// incomplete-round word, so rows that peers may still read stay untouched. +template +__device__ __forceinline__ void prepareCombine(uint8_t* combine_input_base, void const* source_payload, int elements_per_token, int ep_size, int max_tokens_per_rank, uint32_t* flag_val_ptr, int const* recv_counters, - int source_stride_per_token, int workspace_stride_per_token, int prepare_num_tokens) + int source_stride_per_token, int workspace_stride_per_token, int prepare_num_tokens, int ep_rank, + uint32_t const* incomplete_round) { #if TLLM_MOE_A2A_COMPILE_SM90 cudaGridDependencySynchronize(); cudaTriggerProgrammaticLaunchCompletion(); #endif - uint32_t const parity = round_parity(*flag_val_ptr); + // Block 0 advances the flag value concurrently, so this may read the round's dispatch or combine flag value. + uint32_t const flag_value = *flag_val_ptr; + uint32_t const parity = round_parity(flag_value); int const* const round_recv_counters = recv_counters + parity * ep_size; if (blockIdx.x == 0 && threadIdx.x == 0) { @@ -1031,6 +1265,11 @@ __global__ void moeA2APrepareCombineKernel(uint8_t* combine_input_base, void con // Skip invalid tokens beyond per-rank recv count if (local_token_idx >= round_recv_counters[rank_idx]) return; + if constexpr (HAS_INCOMPLETE_ROUND) + { + if (rank_idx != ep_rank && only_own_rows_valid(*incomplete_round, flag_value)) + return; + } size_t const source_offset = static_cast(global_token_idx) * source_stride_per_token; size_t const workspace_offset = static_cast(global_token_idx) * workspace_stride_per_token; @@ -1051,6 +1290,27 @@ __global__ void moeA2APrepareCombineKernel(uint8_t* combine_input_base, void con } } +template +__global__ void moeA2APrepareCombineKernel(uint8_t* combine_input_base, void const* source_payload, + int elements_per_token, int ep_size, int max_tokens_per_rank, uint32_t* flag_val_ptr, int const* recv_counters, + int source_stride_per_token, int workspace_stride_per_token, int prepare_num_tokens) +{ + prepareCombine(combine_input_base, source_payload, elements_per_token, ep_size, + max_tokens_per_rank, flag_val_ptr, recv_counters, source_stride_per_token, workspace_stride_per_token, + prepare_num_tokens, /*ep_rank=*/0, /*incomplete_round=*/nullptr); +} + +template +__global__ void moeA2AIncompleteRoundPrepareCombineKernel(uint8_t* combine_input_base, void const* source_payload, + int elements_per_token, int ep_size, int max_tokens_per_rank, uint32_t* flag_val_ptr, int const* recv_counters, + int source_stride_per_token, int workspace_stride_per_token, int prepare_num_tokens, int ep_rank, + uint32_t const* incomplete_round) +{ + prepareCombine(combine_input_base, source_payload, elements_per_token, ep_size, + max_tokens_per_rank, flag_val_ptr, recv_counters, source_stride_per_token, workspace_stride_per_token, + prepare_num_tokens, ep_rank, incomplete_round); +} + // ============================================================================ // Generic Combine Kernel Implementation (Templated by data type) // ============================================================================ @@ -1063,6 +1323,8 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to int local_token_idx = blockIdx.x; int const size_per_token = elements_per_token * static_cast(sizeof(InputT)); + bool aborted = false; + int only_rank = -1; if (local_num_tokens == 0) { @@ -1083,6 +1345,16 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to cudaGridDependencySynchronize(); #endif + // Load after the grid dependency wait, so that a write of the mask by earlier work is visible. + uint64_t rank_mask[kRankMaskWords] = {}; + if constexpr (ENABLE_RANK_MASK) + { + if (threadIdx.x < warpSize) + { + load_rank_mask_words(ptrs.active_rank_mask, rank_id, rank_mask); + } + } + #if !DISABLE_SYNC_FOR_PROFILING // In-kernel readiness synchronization at start of combine: // - One warp signals readiness to all peers with current flag_val. @@ -1092,6 +1364,15 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to { int lane_id = threadIdx.x % warpSize; uint32_t expected_value = *ptrs.flag_val; + if constexpr (ENABLE_RANK_MASK) + { + uint32_t const incomplete = ptrs.incomplete_round != nullptr ? *ptrs.incomplete_round : 0; + if (only_own_rows_valid(incomplete, expected_value)) + { + keep_only_rank_in_words(rank_mask, rank_id); + only_rank = rank_id; + } + } if (blockIdx.x == 0) { @@ -1102,7 +1383,7 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, peer_rank)) + if (!is_rank_active_in_words(rank_mask, peer_rank)) continue; } uint32_t* flag_addr = &ptrs.completion_flags[peer_rank][rank_id]; @@ -1114,6 +1395,8 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to } } + AbortPoll* abort_poll = start_abort_poll(lane_id); + // Wait for all active peers to signal; skip dead ranks (otherwise we would spin // forever — this is the bug the rank-mask is here to prevent). #pragma unroll 1 // No unroll @@ -1121,13 +1404,21 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to { if constexpr (ENABLE_RANK_MASK) { - if (!is_rank_active(ptrs.active_rank_mask, peer_rank)) + if (!is_rank_active_in_words(rank_mask, peer_rank)) continue; } - if (!wait_round_flag(&ptrs.completion_flags[rank_id][peer_rank], expected_value, ptrs.timeout_cycles, - rank_id, peer_rank, "combine")) + if (!wait_round_flag(&ptrs.completion_flags[rank_id][peer_rank], expected_value, + ptrs.timeout_cycles, ptrs.abort, abort_poll, rank_id, peer_rank, "combine")) { - return; + if constexpr (ENABLE_RANK_MASK) + { + aborted = true; + break; + } + else + { + return; + } } } #if TLLM_MOE_A2A_COMPILE_SM90 @@ -1137,14 +1428,28 @@ __global__ void moeA2ACombineKernel(const CombineKernelPointers ptrs, int max_to asm volatile("fence.acq_rel.sys;" ::: "memory"); #endif } - __syncthreads(); + if constexpr (ENABLE_RANK_MASK) + { + aborted = __syncthreads_or(aborted); + if (aborted && threadIdx.x == 0) + { + // System scope, as the peers mark this word over NVLink with atomicCAS_system (mark_round_incomplete). + atomicCAS_system(ptrs.incomplete_round, 0U, incomplete_round_value(*ptrs.flag_val)); + } + } + else + { + __syncthreads(); + } #endif if (local_num_tokens == 0) return; T* token_output = static_cast(ptrs.output) + local_token_idx * elements_per_token; - vectorized_combine(token_output, size_per_token, stride_per_token, ptrs); + // After an abort, reduce only this rank's own rows, which this round wrote, so that the discarded output is finite. + vectorized_combine( + token_output, size_per_token, stride_per_token, ptrs, aborted ? rank_id : only_rank); #if TLLM_MOE_A2A_COMPILE_SM90 cudaTriggerProgrammaticLaunchCompletion(); #endif @@ -1160,11 +1465,23 @@ void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params) SWITCH_BOOL(params.use_low_precision, LOW_PRECISION, { SWITCH_DTYPE(params.dtype, SrcT, { - auto kernel_fn = moeA2APrepareCombineKernel; - launchWithPdlWhenEnabled("moeA2APrepareCombineKernel", kernel_fn, grid, kBlockSize, 0, params.stream, - combine_input_base, params.source_payload, params.elements_per_token, params.ep_size, - params.max_tokens_per_rank, params.flag_val, params.recv_counters, params.source_stride_per_token, - params.workspace_stride_per_token, params.prepare_num_tokens); + if (params.incomplete_round == nullptr) + { + auto kernel_fn = moeA2APrepareCombineKernel; + launchWithPdlWhenEnabled("moeA2APrepareCombineKernel", kernel_fn, grid, kBlockSize, 0, params.stream, + combine_input_base, params.source_payload, params.elements_per_token, params.ep_size, + params.max_tokens_per_rank, params.flag_val, params.recv_counters, params.source_stride_per_token, + params.workspace_stride_per_token, params.prepare_num_tokens); + } + else + { + auto kernel_fn = moeA2AIncompleteRoundPrepareCombineKernel; + launchWithPdlWhenEnabled("moeA2AIncompleteRoundPrepareCombineKernel", kernel_fn, grid, kBlockSize, 0, + params.stream, combine_input_base, params.source_payload, params.elements_per_token, params.ep_size, + params.max_tokens_per_rank, params.flag_val, params.recv_counters, params.source_stride_per_token, + params.workspace_stride_per_token, params.prepare_num_tokens, params.ep_rank, + params.incomplete_round); + } }); }); } @@ -1183,12 +1500,16 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) TLLM_CHECK(params.ep_rank >= 0 && params.ep_rank < params.ep_size); TLLM_CHECK(params.local_num_tokens >= 0); TLLM_CHECK(params.elements_per_token > 0); - // The local rank must always be marked active in its own view of the mask; - // otherwise the kernel itself would be running on a "dead" rank. if (params.enable_rank_mask) { - TLLM_CHECK_WITH_INFO((params.active_rank_mask[params.ep_rank >> 6] >> (params.ep_rank & 63)) & 1ULL, - "active_rank_mask must mark the local ep_rank (%d) as active", params.ep_rank); + TLLM_CHECK_WITH_INFO( + params.active_rank_mask != nullptr, "rank-mask mode needs the device address of active_rank_mask"); + } + if (params.abort != nullptr) + { + TLLM_CHECK_WITH_INFO( + params.incomplete_round != nullptr, "an abort state needs a workspace laid out with rank_mask"); + TLLM_CHECK_WITH_INFO(params.enable_rank_mask, "the abort state needs enable_rank_mask"); } // Configure kernel launch (one block per token). @@ -1219,16 +1540,14 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) kernel_ptrs.completion_flags[i] = params.completion_flags[i]; } kernel_ptrs.flag_val = params.flag_val; + kernel_ptrs.incomplete_round = params.incomplete_round; // Copy communication tracking pointers kernel_ptrs.topk_target_ranks = params.topk_target_ranks; kernel_ptrs.topk_target_indices = params.topk_target_indices; - // Copy active-rank bitmask into the kernel pointers struct - for (int w = 0; w < kRankMaskWords; ++w) - { - kernel_ptrs.active_rank_mask[w] = params.active_rank_mask[w]; - } + kernel_ptrs.active_rank_mask = params.active_rank_mask; + kernel_ptrs.abort = params.abort; SWITCH_BOOL(params.enable_rank_mask, ENABLE_RANK_MASK, { SWITCH_DTYPE(params.dtype, T, { @@ -1244,9 +1563,36 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) }); } -// Kernel to sanitize expert ids for invalid tokens -__global__ void moeA2ASanitizeExpertIdsKernel(int32_t* expert_ids_ptr, int32_t const* recv_counters_ptr, - uint32_t const* flag_val, int ep_size, int max_tokens_per_rank, int top_k, int32_t invalid_id) +// A peer that may overwrite rows that this rank's fence combines read first stores this rank's incomplete-round word, +// then runs fence.sc.sys (mark_round_incomplete). This kernel runs after those combines in stream order, so the two +// fences decide in Fence-SC order: either the load below returns the word set, or every payload read of the combines +// precedes the overwrite in causality order and cannot return it. +__global__ void moeA2ACheckIncompleteRoundKernel(uint32_t const* incomplete_round, MoeA2AAbortState* abort) +{ + asm volatile("fence.sc.sys;" ::: "memory"); + uint32_t incomplete; + asm volatile("ld.relaxed.sys.u32 %0, [%1];" : "=r"(incomplete) : "l"(incomplete_round) : "memory"); + if (incomplete != 0) + { + record_cancelled(abort); + } +} + +void moe_a2a_check_incomplete_round_launch( + uint32_t const* incomplete_round, MoeA2AAbortState* abort, cudaStream_t stream) +{ + // Without programmatic dependent launch, the kernel starts only after all earlier work in the stream completed. + moeA2ACheckIncompleteRoundKernel<<<1, 1, 0, stream>>>(incomplete_round, abort); +} + +// Sanitize expert ids for invalid tokens. +// With HAS_INCOMPLETE_ROUND, only this rank's own tokens are valid while only_own_rows_valid() holds for the +// incomplete-round word, because a peer that this rank did not see complete may have written a receive counter for rows +// it never finished. +template +__device__ __forceinline__ void sanitizeExpertIds(int32_t* expert_ids_ptr, int32_t const* recv_counters_ptr, + uint32_t const* flag_val, int ep_size, int max_tokens_per_rank, int top_k, int32_t invalid_id, + uint32_t const* incomplete_round, int ep_rank) { int tid = blockIdx.x * blockDim.x + threadIdx.x; int total_tokens = ep_size * max_tokens_per_rank; @@ -1260,8 +1606,14 @@ __global__ void moeA2ASanitizeExpertIdsKernel(int32_t* expert_ids_ptr, int32_t c cudaGridDependencySynchronize(); cudaTriggerProgrammaticLaunchCompletion(); #endif - uint32_t const parity = round_parity(*flag_val); - if (token_idx >= recv_counters_ptr[parity * ep_size + source_rank]) + uint32_t const flag_value = *flag_val; + uint32_t const parity = round_parity(flag_value); + bool invalid = token_idx >= recv_counters_ptr[parity * ep_size + source_rank]; + if constexpr (HAS_INCOMPLETE_ROUND) + { + invalid = invalid || (source_rank != ep_rank && only_own_rows_valid(*incomplete_round, flag_value)); + } + if (invalid) { int32_t* token_expert_ids = expert_ids_ptr + tid * top_k; // Vectorized invalid-id fill: 16B (int4) stores when top_k is a multiple of 4 @@ -1285,14 +1637,39 @@ __global__ void moeA2ASanitizeExpertIdsKernel(int32_t* expert_ids_ptr, int32_t c } } +__global__ void moeA2ASanitizeExpertIdsKernel(int32_t* expert_ids_ptr, int32_t const* recv_counters_ptr, + uint32_t const* flag_val, int ep_size, int max_tokens_per_rank, int top_k, int32_t invalid_id) +{ + sanitizeExpertIds(expert_ids_ptr, recv_counters_ptr, flag_val, ep_size, max_tokens_per_rank, top_k, + invalid_id, /*incomplete_round=*/nullptr, /*ep_rank=*/0); +} + +__global__ void moeA2AIncompleteRoundSanitizeExpertIdsKernel(int32_t* expert_ids_ptr, int32_t const* recv_counters_ptr, + uint32_t const* flag_val, int ep_size, int max_tokens_per_rank, int top_k, int32_t invalid_id, + uint32_t const* incomplete_round, int ep_rank) +{ + sanitizeExpertIds(expert_ids_ptr, recv_counters_ptr, flag_val, ep_size, max_tokens_per_rank, top_k, + invalid_id, incomplete_round, ep_rank); +} + void moe_a2a_sanitize_expert_ids_launch(int32_t* expert_ids, int32_t const* recv_counters, uint32_t const* flag_val, - int32_t invalid_id, int ep_size, int max_tokens_per_rank, int top_k, cudaStream_t stream) + uint32_t const* incomplete_round, int32_t invalid_id, int ep_rank, int ep_size, int max_tokens_per_rank, int top_k, + cudaStream_t stream) { constexpr int kBlockSize = 256; int total_tokens = ep_size * max_tokens_per_rank; int grid = ceilDiv(total_tokens, kBlockSize); - launchWithPdlWhenEnabled("moeA2ASanitizeExpertIdsKernel", moeA2ASanitizeExpertIdsKernel, grid, kBlockSize, 0, - stream, expert_ids, recv_counters, flag_val, ep_size, max_tokens_per_rank, top_k, invalid_id); + if (incomplete_round == nullptr) + { + launchWithPdlWhenEnabled("moeA2ASanitizeExpertIdsKernel", moeA2ASanitizeExpertIdsKernel, grid, kBlockSize, 0, + stream, expert_ids, recv_counters, flag_val, ep_size, max_tokens_per_rank, top_k, invalid_id); + } + else + { + launchWithPdlWhenEnabled("moeA2AIncompleteRoundSanitizeExpertIdsKernel", + moeA2AIncompleteRoundSanitizeExpertIdsKernel, grid, kBlockSize, 0, stream, expert_ids, recv_counters, + flag_val, ep_size, max_tokens_per_rank, top_k, invalid_id, incomplete_round, ep_rank); + } } } // namespace kernels::moe_comm diff --git a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h index ce9ba8ef31ed..08b437ad4eb0 100644 --- a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h +++ b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h @@ -17,6 +17,7 @@ #pragma once #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/tllmDataType.h" +#include #include #include @@ -37,6 +38,42 @@ static_assert(kRankMaskWords * 64 >= kMaxRanks, "active_rank_mask too small for static constexpr int64_t kAssumedClockHz = 2000ll * 1000ll * 1000ll; static constexpr int64_t kDefaultTimeoutCycles = 300ll * kAssumedClockHz; +// Waits on a peer shorter than kAbortPollCycles never read the host abort state. +// Longer waits read it once per kAbortPollCycles per block. +static constexpr int64_t kAbortPollCycles = 100ll * 1000ll; + +// A wait on a peer longer than kPublishWaitedFlagCycles publishes the flag value it waits for in +// MoeA2AAbortState::waited_flag, so the host can tell which peer this rank waits on. +static constexpr int64_t kPublishWaitedFlagCycles = 20ll * 1000ll * 1000ll; + +//! \brief Abort state of the rank-mask collectives, in pinned host memory. +//! +//! The host sets `requested` to end the waits on peers. A kernel that sees it stops waiting and sets `status` to +//! kCancelled. While `requested` or `status` is nonzero, the outputs of every collective that uses the abort state +//! are invalid. The incomplete-round word below decides what the following collectives exchange. +struct MoeA2AAbortState +{ + static constexpr int32_t kCancelled = 1; + + int32_t requested; + // kCancelled once a kernel stopped waiting because of `requested`, or the incomplete-round check found the word + // set, 0 otherwise + int32_t status; + // Per peer, the round flag value that a long wait on the peer waits for, 0 otherwise + uint32_t waited_flag[kMaxRanks]; +}; + +// Incomplete-round word: a uint32_t in the workspace of each rank of a layout built with a rank mask. A collective +// that ends without the data of some active source stores its incomplete-round value (its round flag value with bit 0 +// set, so never 0) there, unless the word is already nonzero. The word keeps that value until the workspace is reset. +// While it is nonzero, every later collective of this rank exchanges with this rank only, so no peer reads rows of an +// incomplete round. The MoE after a dispatch may still overwrite rows of this rank that an active peer's fence combine +// reads. So a fence dispatch that aborts, or that starts with the word set, first stores the value in the word of +// every active rank, and a check after each forward that finds its word set marks the abort state cancelled +// (moe_a2a_check_incomplete_round_launch). The incomplete-round value of round N's combine, which round N + 1's +// dispatch shares, is ignored within round N: this rank's combine or a peer's next dispatch may store it during the +// round, and the prepare, the sanitize and every block of the combine must exchange with the same peers. + // Describes a single payload type to be communicated struct PayloadDescriptor { @@ -76,8 +113,15 @@ struct DispatchKernelPointers // Active-rank bitmask: bit i set => rank i is alive and participates in this collective. // Word w covers ranks 64*w..64*w+63 for w in [0, kRankMaskWords). Tokens routed to a masked // rank are dropped (topk_*[k] = -1); flag writes/waits to/from masked peers are skipped. - // The local rank's own bit must always be set; this is checked at launch time. - uint64_t active_rank_mask[kRankMaskWords]; + // The address is fixed, so captured CUDA graphs read a mask rewritten in place while no collective runs. + uint64_t const* active_rank_mask{nullptr}; + // Byte offset from completion_flags[rank] to the incomplete-round word (see above) of that rank, or 0 without one. + // Every rank's workspace has the same layout, so one offset locates the word of every rank. + int64_t incomplete_round_offset{0}; + MoeA2AAbortState* abort{nullptr}; + // The rank-mask fields above fill kRankMaskWords words, so the parameter offsets of the fields and of the kernel + // arguments after them do not depend on them. + uint64_t rank_mask_reserved{0}; // Host-selected wait budget in clock64() cycles. int64_t timeout_cycles{kDefaultTimeoutCycles}; @@ -101,12 +145,28 @@ struct CombineKernelPointers // Active-rank bitmask: see DispatchKernelPointers::active_rank_mask. Combine skips // completion flag writes/waits to/from inactive peers. - uint64_t active_rank_mask[kRankMaskWords]; + uint64_t const* active_rank_mask{nullptr}; + // Incomplete-round word in the local workspace (see above), or nullptr without one. + uint32_t* incomplete_round{nullptr}; + MoeA2AAbortState* abort{nullptr}; + // The rank-mask fields above fill kRankMaskWords words, as in DispatchKernelPointers. + uint64_t rank_mask_reserved{0}; // Host-selected wait budget in clock64() cycles. int64_t timeout_cycles{kDefaultTimeoutCycles}; }; +// The kernels take these structs by value, followed by scalar arguments, so the rank-mask fields keep a fixed +// kRankMaskWords-word slot and timeout_cycles stays the last field. The parameter offsets of timeout_cycles and of +// the scalar arguments do not depend on the rank-mask fields, and neither does the generated code of the kernels +// without a rank mask. +static_assert(offsetof(DispatchKernelPointers, timeout_cycles) + == offsetof(DispatchKernelPointers, active_rank_mask) + kRankMaskWords * sizeof(uint64_t)); +static_assert(sizeof(DispatchKernelPointers) == offsetof(DispatchKernelPointers, timeout_cycles) + sizeof(int64_t)); +static_assert(offsetof(CombineKernelPointers, timeout_cycles) + == offsetof(CombineKernelPointers, active_rank_mask) + kRankMaskWords * sizeof(uint64_t)); +static_assert(sizeof(CombineKernelPointers) == offsetof(CombineKernelPointers, timeout_cycles) + sizeof(int64_t)); + // Dispatch phase parameters struct MoeA2ADispatchParams { @@ -142,6 +202,8 @@ struct MoeA2ADispatchParams int* recv_counters[kMaxRanks]; uint32_t* completion_flags[kMaxRanks]; // If completion_flags[target_rank][source_rank] == *flag_val, then source // rank has signaled the target rank + // Byte offset from completion_flags[rank] to the incomplete-round word (see above) of that rank, or 0 without one. + int64_t incomplete_round_offset{0}; void* recv_buffers[kMaxRanks][kMaxPayloads]; // Per-rank receive buffers for each payload // Optional: Statistics for EPLB @@ -150,15 +212,12 @@ struct MoeA2ADispatchParams int const* eplb_local_stats; // [eplb_stats_num_experts] int* eplb_gathered_stats[kMaxRanks]; // [ep_size, eplb_stats_num_experts] per rank - // Whether to instantiate a kernel with active-rank checks. - // This is a launch-lifetime mode, independent of future execution-abort handling. + // Whether to instantiate a kernel with active-rank checks. An abort state requires it. bool enable_rank_mask{false}; - // Active-rank bitmask: see DispatchKernelPointers::active_rank_mask. Used only when - // enable_rank_mask is true; defaults to all-ones for backwards-compatible behavior. - // The mask is copied by value into kernel arguments. Rank-mask mode must reject - // CUDA graph replay until generation-scoped invalidation and recapture are available. - uint64_t active_rank_mask[kRankMaskWords] = {~uint64_t{0}, ~uint64_t{0}, ~uint64_t{0}, ~uint64_t{0}}; + // Active-rank bitmask: see DispatchKernelPointers::active_rank_mask. Required when enable_rank_mask is true. + uint64_t const* active_rank_mask{nullptr}; + MoeA2AAbortState* abort{nullptr}; // Host-selected wait budget in clock64() cycles. int64_t timeout_cycles{kDefaultTimeoutCycles}; @@ -205,6 +264,8 @@ struct MoeA2ACombineParams // per k, -1 for duplicates int* topk_target_indices; // Top-K compact routing info per local token (size: [local_num_tokens, top_k]), dst index // per k, -1 for duplicates + // Incomplete-round word in the local workspace (see above), or nullptr without one. + uint32_t* incomplete_round{nullptr}; // Local recv_counters[parity][source_rank]. The two parity banks alternate // between A2A rounds. int const* recv_counters; @@ -214,15 +275,12 @@ struct MoeA2ACombineParams // rank has signaled the target rank uint8_t* combine_input_buffers[kMaxRanks]; // Expert-output/staging region on each rank - // Whether to instantiate a kernel with active-rank checks in peer synchronization. - // This is a launch-lifetime mode, independent of future execution-abort handling. + // Whether to instantiate a kernel with active-rank checks in peer synchronization. An abort state requires it. bool enable_rank_mask{false}; - // Active-rank bitmask: see DispatchKernelPointers::active_rank_mask. Used only when - // enable_rank_mask is true; defaults to all-ones for backwards-compatible behavior. - // The mask is copied by value into kernel arguments. Rank-mask mode must reject - // CUDA graph replay until generation-scoped invalidation and recapture are available. - uint64_t active_rank_mask[kRankMaskWords] = {~uint64_t{0}, ~uint64_t{0}, ~uint64_t{0}, ~uint64_t{0}}; + // Active-rank bitmask: see DispatchKernelPointers::active_rank_mask. Required when enable_rank_mask is true. + uint64_t const* active_rank_mask{nullptr}; + MoeA2AAbortState* abort{nullptr}; // Host-selected wait budget in clock64() cycles. int64_t timeout_cycles{kDefaultTimeoutCycles}; @@ -236,12 +294,20 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params); void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params); +// Mark `abort` cancelled if the incomplete-round word (see above) is set, ordered after every earlier fence combine in +// `stream`. Launch once per forward, after its last combine. +void moe_a2a_check_incomplete_round_launch( + uint32_t const* incomplete_round, MoeA2AAbortState* abort, cudaStream_t stream); + // Sanitize expert IDs for invalid tokens // expert_ids: [ep_size, max_tokens_per_rank, top_k] (int32) // recv_counters: [2, ep_size] (int32), number of valid tokens per source +// incomplete_round: incomplete-round word (see above), or nullptr. While it is nonzero, only the tokens from ep_rank +// are valid. // invalid_id: value to fill for invalid tokens' expert ids void moe_a2a_sanitize_expert_ids_launch(int32_t* expert_ids, int32_t const* recv_counters, uint32_t const* flag_val, - int32_t invalid_id, int ep_size, int max_tokens_per_rank, int top_k, cudaStream_t stream); + uint32_t const* incomplete_round, int32_t invalid_id, int ep_rank, int ep_size, int max_tokens_per_rank, int top_k, + cudaStream_t stream); } // namespace kernels::moe_comm diff --git a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h index 6c9adb37eaff..669f7914665f 100644 --- a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h +++ b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h @@ -21,6 +21,7 @@ #include "tensorrt_llm/kernels/moe/communication/moeAlltoAllSentinel.h" #include +#include #include #include #include @@ -101,8 +102,11 @@ enum MoeA2AMetaInfoIndex : int64_t // Reserved receive capacity, including payload alignment padding. SENTINEL_RECV_SIZE_INDEX = 25, + // Offset of the incomplete-round word (see moeAlltoAllKernels.h), 0 in layouts without a rank mask. + INCOMPLETE_ROUND_OFFSET_INDEX = 26, + // Number of int64_t entries in the metadata tensor; not a stored layout field. - NUM_METAINFO_FIELDS = 26 + NUM_METAINFO_FIELDS = 27 }; using MoeA2AWorkspaceLayout = std::array; @@ -138,6 +142,7 @@ inline std::vector> getMoeA2AMetaInfoIndexPairs( {"MOE_A2A_SENTINEL_COMBINE_LANDING_OFFSET_INDEX", SENTINEL_COMBINE_LANDING_OFFSET_INDEX}, {"MOE_A2A_SENTINEL_RECV_OFFSET_INDEX", SENTINEL_RECV_OFFSET_INDEX}, {"MOE_A2A_SENTINEL_RECV_SIZE_INDEX", SENTINEL_RECV_SIZE_INDEX}, + {"MOE_A2A_INCOMPLETE_ROUND_OFFSET_INDEX", INCOMPLETE_ROUND_OFFSET_INDEX}, {"MOE_A2A_NUM_METAINFO_FIELDS", NUM_METAINFO_FIELDS}, {"MOE_A2A_MAX_RANKS", kMaxRanks}, {"MOE_A2A_MAX_TOP_K", kMaxTopK}, @@ -151,6 +156,8 @@ inline std::vector> getMoeA2AMetaInfoIndexPairs( {"MOE_A2A_SENTINEL_FORMAT_MXFP8", static_cast(sentinel::PayloadFormat::kMxfp8)}, {"MOE_A2A_SENTINEL_FORMAT_FP8_BLOCK128", static_cast(sentinel::PayloadFormat::kFp8Block128)}, {"MOE_A2A_SENTINEL_FORMAT_NVFP4", static_cast(sentinel::PayloadFormat::kNvfp4)}, + {"MOE_A2A_ABORT_STATE_WORDS", sizeof(MoeA2AAbortState) / sizeof(int32_t)}, + {"MOE_A2A_ABORT_WAITED_FLAG_INDEX", offsetof(MoeA2AAbortState, waited_flag) / sizeof(int32_t)}, }; } diff --git a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp index 4af9c20c6957..6799fa19b278 100644 --- a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp @@ -112,15 +112,22 @@ MoeA2AWorkspaceLayout const& readWorkspaceLayout(torch::Tensor const& metainfo) return *reinterpret_cast(metainfo.data_ptr()); } +// calculateWorkspaceLayout reserves the flag value first, at offset 0, so only a layout that holds the incomplete-round +// word has a nonzero offset for it. +inline bool hasIncompleteRoundWord(MoeA2AWorkspaceLayout const& layout) +{ + return layout[INCOMPLETE_ROUND_OFFSET_INDEX] != 0; +} + inline bool hasActiveRankMask(torch::optional const& maskTensor) { return maskTensor.has_value() && maskTensor.value().defined(); } -// Resolve a provided rank-mask tensor into a fixed-width uint64 array. On failure +// Return the device address of a provided rank-mask tensor, which the kernels read when they run. On failure // (wrong dtype / device / shape), throw at the Python op boundary rather than launch. -inline void resolveActiveRankMask(torch::optional const& maskTensor, int64_t epRank, - uint64_t (&out)[tensorrt_llm::kernels::moe_comm::kRankMaskWords]) +inline uint64_t const* resolveActiveRankMask( + torch::optional const& maskTensor, int64_t epRank, torch::Tensor const& workspace) { using tensorrt_llm::kernels::moe_comm::kRankMaskWords; using tensorrt_llm::kernels::moe_comm::kMaxRanks; @@ -128,25 +135,36 @@ inline void resolveActiveRankMask(torch::optional const& maskTens epRank >= 0 && epRank < kMaxRanks, "epRank must be in the range [0, ", kMaxRanks, ") for active_rank_mask"); TORCH_CHECK(hasActiveRankMask(maskTensor), "active_rank_mask must be defined"); torch::Tensor const& t = maskTensor.value(); - TORCH_CHECK(t.is_cpu(), "active_rank_mask must be a CPU tensor"); + TORCH_CHECK(t.device() == workspace.device(), "active_rank_mask must be on the workspace device"); TORCH_CHECK(t.scalar_type() == torch::kUInt64, "active_rank_mask must have dtype uint64"); TORCH_CHECK(t.dim() == 1, "active_rank_mask must be a 1D tensor"); TORCH_CHECK(t.numel() == kRankMaskWords, "active_rank_mask must have exactly ", kRankMaskWords, " uint64 elements"); TORCH_CHECK(t.is_contiguous(), "active_rank_mask must be contiguous"); - auto const* src = static_cast(t.const_data_ptr()); - for (int w = 0; w < kRankMaskWords; ++w) + return static_cast(t.const_data_ptr()); +} + +// The kernels read the host abort state through unified virtual addressing, so the caller allocates it in pinned +// memory. +inline tensorrt_llm::kernels::moe_comm::MoeA2AAbortState* resolveAbortState(torch::optional const& abort) +{ + using tensorrt_llm::kernels::moe_comm::MoeA2AAbortState; + if (!abort.has_value()) { - out[w] = src[w]; + return nullptr; } - // Local rank's bit must be set; otherwise the kernel would be running on a "dead" rank. - TORCH_CHECK((out[epRank >> 6] >> (epRank & 63)) & 1ULL, "active_rank_mask must mark the local ep_rank (", epRank, - ") as active"); + torch::Tensor const& t = abort.value(); + constexpr int64_t kWords = sizeof(MoeA2AAbortState) / sizeof(int32_t); + TORCH_CHECK( + t.is_cpu() && t.scalar_type() == torch::kInt32 && t.dim() == 1 && t.numel() == kWords && t.is_contiguous(), + "abort_state must be a contiguous int32 CPU tensor of ", kWords, " elements"); + TORCH_CHECK(t.is_pinned(), "abort_state must be in pinned memory, because the kernels read it at its host address"); + return static_cast(t.data_ptr()); } // All offsets and capacities are per rank. Payload capacities are padded so // the following phase's control buffers remain aligned. MoeA2AWorkspaceLayout calculateWorkspaceLayout(int64_t epSize, int64_t maxNumTokens, int64_t topK, - int64_t dispatchBytes, int64_t combineInputBytes, int64_t eplbStatsNumExperts) + int64_t dispatchBytes, int64_t combineInputBytes, int64_t eplbStatsNumExperts, bool rankMask) { using tensorrt_llm::kernels::moe_comm::kMaxRanks; using tensorrt_llm::kernels::moe_comm::kMaxTopK; @@ -181,6 +199,10 @@ MoeA2AWorkspaceLayout calculateWorkspaceLayout(int64_t epSize, int64_t maxNumTok int64_t const rankCountsBytes = epSize * sizeof(int32_t); reserve(FLAG_VAL_OFFSET_INDEX, sizeof(uint32_t)); + if (rankMask) + { + reserve(INCOMPLETE_ROUND_OFFSET_INDEX, sizeof(uint32_t)); + } // Dispatch writes these counters and routes; combine reuses the completed routes. reserve(LOCAL_TOKEN_COUNTER_OFFSET_INDEX, sizeof(int32_t)); @@ -207,10 +229,10 @@ MoeA2AWorkspaceLayout calculateWorkspaceLayout(int64_t epSize, int64_t maxNumTok } torch::Tensor moeA2AGetWorkspaceLayoutOp(int64_t epSize, int64_t maxNumTokens, int64_t topK, int64_t dispatchBytes, - int64_t combineInputBytes, torch::optional eplbStatsNumExperts) + int64_t combineInputBytes, torch::optional eplbStatsNumExperts, bool rankMask) { auto const layout = calculateWorkspaceLayout( - epSize, maxNumTokens, topK, dispatchBytes, combineInputBytes, eplbStatsNumExperts.value_or(0)); + epSize, maxNumTokens, topK, dispatchBytes, combineInputBytes, eplbStatsNumExperts.value_or(0), rankMask); auto metainfo = torch::empty({NUM_METAINFO_FIELDS}, torch::TensorOptions().dtype(torch::kInt64).device(torch::kCPU)); std::copy(layout.begin(), layout.end(), metainfo.data_ptr()); @@ -454,7 +476,8 @@ void moeA2ASentinelExtendWorkspaceLayoutOp( { auto layout = readWorkspaceLayout(metainfo); auto const base = calculateWorkspaceLayout(layout[EP_SIZE_INDEX], layout[MAX_NUM_TOKENS_INDEX], layout[TOP_K_INDEX], - layout[DISPATCH_PAYLOAD_SIZE_INDEX], layout[COMBINE_INPUT_SIZE_INDEX], layout[EPLB_STATS_NUM_EXPERTS_INDEX]); + layout[DISPATCH_PAYLOAD_SIZE_INDEX], layout[COMBINE_INPUT_SIZE_INDEX], layout[EPLB_STATS_NUM_EXPERTS_INDEX], + hasIncompleteRoundWord(layout)); TORCH_CHECK(layout == base, "metainfo must be a layout from moe_a2a_get_workspace_layout"); appendSentinelRegions(layout, sentinelMaxTokens, payloadFormat, hiddenSize); std::copy(layout.begin(), layout.end(), metainfo.data_ptr()); @@ -477,7 +500,10 @@ bool moeA2ASentinelAvailableOp() } // Initialize control state after allocation; layout construction itself is CPU-only. -void moeA2AInitializeOp(torch::Tensor const& workspace, torch::Tensor const& metainfo, int64_t epRank, int64_t epSize) +// The op resets only the local rank's row. With barrier=false it skips the cross-rank barrier, and the caller must +// synchronize the ranks before any peer writes into the row. +void moeA2AInitializeOp( + torch::Tensor const& workspace, torch::Tensor const& metainfo, int64_t epRank, int64_t epSize, bool barrier) { CHECK_TH_CUDA(workspace); CHECK_TYPE(workspace, torch::kUInt8); @@ -487,7 +513,8 @@ void moeA2AInitializeOp(torch::Tensor const& workspace, torch::Tensor const& met auto const& layout = readWorkspaceLayout(metainfo); TORCH_CHECK(layout[EP_SIZE_INDEX] == epSize, "Workspace layout EP size mismatch"); auto expected = calculateWorkspaceLayout(epSize, layout[MAX_NUM_TOKENS_INDEX], layout[TOP_K_INDEX], - layout[DISPATCH_PAYLOAD_SIZE_INDEX], layout[COMBINE_INPUT_SIZE_INDEX], layout[EPLB_STATS_NUM_EXPERTS_INDEX]); + layout[DISPATCH_PAYLOAD_SIZE_INDEX], layout[COMBINE_INPUT_SIZE_INDEX], layout[EPLB_STATS_NUM_EXPERTS_INDEX], + hasIncompleteRoundWord(layout)); bool const hasSentinel = layout[SENTINEL_MAX_NUM_TOKENS_INDEX] > 0; if (hasSentinel) { @@ -514,7 +541,10 @@ void moeA2AInitializeOp(torch::Tensor const& workspace, torch::Tensor const& met // NVLinkOneSided strategy unusable there; fall back to the Torch process // group the Ray workers already initialise, as pg_utils does elsewhere. cudaDeviceSynchronize(); - moeA2ABarrier(); + if (barrier) + { + moeA2ABarrier(); + } } // ============================================================================ @@ -674,10 +704,17 @@ void sentinelCombine(torch::Tensor const& payload, int64_t localNumTokens, torch // - numExperts: Total number of routing slots (tokenSelectedExperts values are in [0, numExperts)) // - eplbStatsNumExperts: Number of experts used for EPLB stats (may be <= numExperts) // - eplbLocalStats: [eplbStatsNumExperts] tensor containing local statistics for EPLB. +// - enableRankMask: Exchange only with the ranks set in activeRankMask. The workspace layout must be built with +// rank_mask=True. +// - activeRankMask: [kRankMaskWords] uint64 tensor on the workspace device. The kernels read it at execution time, +// so a captured CUDA graph follows later updates. The kernels treat the local rank's bit as set. // - useSentinel: run the sentinel transport; the payloads must be the hidden states, their scales (quantized // formats), the expert ids and optionally the weights, in the workspace's sentinel payload format, and // expertIdPayloadIndex / invalidTokenExpertId must be given. Received rows are then (source rank, token), not // compacted, and rows of tokens not routed here carry invalidTokenExpertId. +// - abortState: (Optional) pinned int32 host tensor holding a MoeA2AAbortState, for enableRankMask without +// useSentinel. Long waits on peers publish the flag value they wait for in it. Setting its requested word stops the +// waits on peers. // // Return values: // - recvTensors: Vector of receive buffers (one tensor per payload), each [ep_size, runtimeMaxTokensPerRank, @@ -693,7 +730,7 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( torch::Tensor const& workspace, torch::Tensor const& metainfo, int64_t runtimeMaxTokensPerRank, int64_t epRank, int64_t epSize, int64_t topK, int64_t numExperts, torch::optional eplbLocalStats, torch::optional expertIdPayloadIndex, torch::optional invalidTokenExpertId, bool enableRankMask, - torch::optional activeRankMask, bool useSentinel) + torch::optional activeRankMask, bool useSentinel, torch::optional abortState) { using tensorrt_llm::kernels::moe_comm::PayloadDescriptor; using tensorrt_llm::kernels::moe_comm::MoeA2ADispatchParams; @@ -848,8 +885,8 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( { TORCH_CHECK(!enableEplb, "A sentinel dispatch does not gather EPLB statistics"); // The sentinel kernels never drain or re-arm a masked peer's rows; fault tolerance uses fence. - TORCH_CHECK( - !enableRankMask && !hasActiveRankMask(activeRankMask), "A sentinel dispatch does not take a rank mask"); + TORCH_CHECK(!enableRankMask && !hasActiveRankMask(activeRankMask) && !abortState.has_value(), + "A sentinel dispatch does not take a rank mask or an abort state"); TORCH_CHECK(sanitizeExpertIds, "A sentinel dispatch needs expert_id_payload_index and invalid_token_expert_id: it fills the expert ids of " "rows without a token"); @@ -894,6 +931,9 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( params.send_counters = reinterpret_cast(rankWorkSpacePtr + offsets[SEND_COUNTERS_OFFSET_INDEX]); params.topk_target_ranks = reinterpret_cast(rankWorkSpacePtr + offsets[TOPK_TARGET_RANKS_OFFSET_INDEX]); params.topk_target_indices = reinterpret_cast(rankWorkSpacePtr + offsets[TOPK_TARGET_INDICES_OFFSET_INDEX]); + params.incomplete_round_offset = hasIncompleteRoundWord(offsets) + ? offsets[INCOMPLETE_ROUND_OFFSET_INDEX] - offsets[DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX] + : 0; for (int target_rank = 0; target_rank < epSize; target_rank++) { @@ -929,17 +969,17 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( params.eplb_local_stats = nullptr; } - // Resolve the optional active-rank mask. Default (no mask) = all bits set, which - // exactly reproduces the pre-fault-tolerance kernel behavior. + // Rank-mask mode reads the mask at its device address; without it, the mask-free kernels run. params.enable_rank_mask = enableRankMask; if (params.enable_rank_mask) { - resolveActiveRankMask(activeRankMask, epRank, params.active_rank_mask); + params.active_rank_mask = resolveActiveRankMask(activeRankMask, epRank, workspace); } else { TORCH_CHECK(!hasActiveRankMask(activeRankMask), "active_rank_mask requires enable_rank_mask=True"); } + params.abort = resolveAbortState(abortState); params.stream = at::cuda::getCurrentCUDAStream(); params.timeout_cycles = gTimeoutCycles.load(std::memory_order_relaxed); @@ -982,7 +1022,8 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumTokens, torch::Tensor const& workspace, torch::Tensor const& metainfo, int64_t runtimeMaxTokensPerRank, int64_t epRank, int64_t epSize, int64_t topK, int64_t combinePayloadOffset, bool payloadInWorkspace, bool useLowPrecision = false, bool enableRankMask = false, - torch::optional activeRankMask = torch::nullopt, bool useSentinel = false) + torch::optional activeRankMask = torch::nullopt, bool useSentinel = false, + torch::optional abortState = torch::nullopt) { using tensorrt_llm::kernels::moe_comm::MoeA2ACombineParams; using tensorrt_llm::kernels::moe_comm::moe_a2a_combine_launch; @@ -1071,8 +1112,8 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke if (useSentinel) { TORCH_CHECK(!useLowPrecision, "A sentinel combine does not use the FP8 wire format"); - TORCH_CHECK( - !enableRankMask && !hasActiveRankMask(activeRankMask), "A sentinel combine does not take a rank mask"); + TORCH_CHECK(!enableRankMask && !hasActiveRankMask(activeRankMask) && !abortState.has_value(), + "A sentinel combine does not take a rank mask or an abort state"); sentinelCombine( payload, localNumTokens, workspace, offsets, runtimeMaxTokensPerRank, epRank, epSize, topK, output); cudaError_t result = cudaGetLastError(); @@ -1101,6 +1142,9 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke = useLowPrecision && !inputIsWorkspace ? static_cast(elementsPerToken) : params.source_stride_per_token; params.flag_val = reinterpret_cast(rankWorkSpacePtr + offsets[FLAG_VAL_OFFSET_INDEX]); + params.incomplete_round = hasIncompleteRoundWord(offsets) + ? reinterpret_cast(rankWorkSpacePtr + offsets[INCOMPLETE_ROUND_OFFSET_INDEX]) + : nullptr; params.topk_target_ranks = reinterpret_cast(rankWorkSpacePtr + offsets[TOPK_TARGET_RANKS_OFFSET_INDEX]); params.topk_target_indices = reinterpret_cast(rankWorkSpacePtr + offsets[TOPK_TARGET_INDICES_OFFSET_INDEX]); params.recv_counters = reinterpret_cast(rankWorkSpacePtr + offsets[RECV_COUNTERS_OFFSET_INDEX]); @@ -1113,16 +1157,17 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke params.combine_input_buffers[target_rank] = target_workspace_ptr + combinePayloadOffset; } - // Resolve the optional active-rank mask. Default (no mask) = all bits set. + // Rank-mask mode reads the mask at its device address; without it, the mask-free kernels run. params.enable_rank_mask = enableRankMask; if (params.enable_rank_mask) { - resolveActiveRankMask(activeRankMask, epRank, params.active_rank_mask); + params.active_rank_mask = resolveActiveRankMask(activeRankMask, epRank, workspace); } else { TORCH_CHECK(!hasActiveRankMask(activeRankMask), "active_rank_mask requires enable_rank_mask=True"); } + params.abort = resolveAbortState(abortState); // Resolve the complete payload plan once. Prepare always launches at least one block to // advance flag_val, but prepare_num_tokens=0 performs no payload work. @@ -1164,10 +1209,33 @@ void moeA2ASanitizeExpertIdsOp(torch::Tensor& expert_ids, torch::Tensor& workspa uint8_t* rankWorkSpacePtr = workspace.data_ptr() + epRank * workspace.stride(0); int* recv_counters = reinterpret_cast(rankWorkSpacePtr + offsets[RECV_COUNTERS_OFFSET_INDEX]); uint32_t* flag_val = reinterpret_cast(rankWorkSpacePtr + offsets[FLAG_VAL_OFFSET_INDEX]); + uint32_t const* incomplete_round = hasIncompleteRoundWord(offsets) + ? reinterpret_cast(rankWorkSpacePtr + offsets[INCOMPLETE_ROUND_OFFSET_INDEX]) + : nullptr; tensorrt_llm::kernels::moe_comm::moe_a2a_sanitize_expert_ids_launch(expert_ids.data_ptr(), recv_counters, - flag_val, static_cast(invalid_expert_id), ep_size, runtime_max_tokens_per_rank, top_k, - at::cuda::getCurrentCUDAStream()); + flag_val, incomplete_round, static_cast(invalid_expert_id), static_cast(epRank), ep_size, + runtime_max_tokens_per_rank, top_k, at::cuda::getCurrentCUDAStream()); +} + +// Op: moe_a2a_check_incomplete_round +void moeA2ACheckIncompleteRoundOp( + torch::Tensor const& workspace, torch::Tensor const& metainfo, int64_t epRank, torch::Tensor& abortState) +{ + CHECK_TH_CUDA(workspace); + CHECK_TYPE(workspace, torch::kUInt8); + TORCH_CHECK(workspace.dim() == 2 && epRank >= 0 && epRank < workspace.size(0), + "workspace must be [ep_size, size_per_rank] and contain epRank"); + auto const& offsets = readWorkspaceLayout(metainfo); + TORCH_CHECK(hasIncompleteRoundWord(offsets), "the workspace layout has no incomplete-round word (rank_mask)"); + + uint8_t const* rankWorkspace = workspace.const_data_ptr() + epRank * workspace.stride(0); + tensorrt_llm::kernels::moe_comm::moe_a2a_check_incomplete_round_launch( + reinterpret_cast(rankWorkspace + offsets[INCOMPLETE_ROUND_OFFSET_INDEX]), + resolveAbortState(abortState), at::cuda::getCurrentCUDAStream()); + cudaError_t result = cudaGetLastError(); + TORCH_CHECK( + result == cudaSuccess, "moe_a2a_check_incomplete_round kernel launch failed: ", cudaGetErrorString(result)); } // Return a workspace-backed tensor for combine payload region using from_blob @@ -1224,7 +1292,8 @@ TORCH_LIBRARY_FRAGMENT(trtllm, module) "int? invalid_token_expert_id=None, " "bool enable_rank_mask=False, " "Tensor? active_rank_mask=None, " - "bool use_sentinel=False) -> (Tensor(a!)[], int, Tensor(a!))"); + "bool use_sentinel=False, " + "Tensor(b!)? abort_state=None) -> (Tensor(a!)[], int, Tensor(a!))"); module.def( "moe_a2a_combine(Tensor(a) payload, int local_num_tokens," "Tensor(a!) workspace, Tensor metainfo, int runtime_max_tokens_per_rank, " @@ -1233,11 +1302,15 @@ TORCH_LIBRARY_FRAGMENT(trtllm, module) "bool use_low_precision=False, " "bool enable_rank_mask=False, " "Tensor? active_rank_mask=None, " - "bool use_sentinel=False) -> Tensor"); - module.def("moe_a2a_initialize(Tensor(a!) workspace, Tensor metainfo, int ep_rank, int ep_size) -> ()"); + "bool use_sentinel=False, " + "Tensor(b!)? abort_state=None) -> Tensor"); + module.def( + "moe_a2a_initialize(Tensor(a!) workspace, Tensor metainfo, int ep_rank, int ep_size, bool barrier=True) -> ()"); module.def( "moe_a2a_sanitize_expert_ids(Tensor(a!) expert_ids, Tensor(a!) workspace, Tensor metainfo, int ep_rank, int " "invalid_expert_id) -> ()"); + module.def( + "moe_a2a_check_incomplete_round(Tensor workspace, Tensor metainfo, int ep_rank, Tensor(a!) abort_state) -> ()"); module.def( "moe_a2a_get_combine_payload_tensor(Tensor(a) workspace, Tensor metainfo, int ep_rank, int ep_size, " "int runtime_max_tokens_per_rank, ScalarType out_dtype, int hidden_size) -> Tensor(a)"); @@ -1245,7 +1318,8 @@ TORCH_LIBRARY_FRAGMENT(trtllm, module) module.def("moe_a2a_get_timeout() -> int", &tensorrt_llm::torch_ext::moe_comm::moeA2AGetTimeoutOp); module.def( "moe_a2a_get_workspace_layout(int ep_size, int max_num_tokens_per_rank, int top_k, " - "int dispatch_payload_bytes, int combine_input_bytes, int? eplb_stats_num_experts=None) -> Tensor", + "int dispatch_payload_bytes, int combine_input_bytes, int? eplb_stats_num_experts=None, " + "bool rank_mask=False) -> Tensor", &tensorrt_llm::torch_ext::moe_comm::moeA2AGetWorkspaceLayoutOp); module.def( "moe_a2a_sentinel_extend_workspace_layout(Tensor(a!) metainfo, int sentinel_max_tokens, int payload_format, " @@ -1262,6 +1336,7 @@ TORCH_LIBRARY_IMPL(trtllm, CUDA, module) module.impl("moe_a2a_combine", &tensorrt_llm::torch_ext::moe_comm::moeA2ACombineOp); module.impl("moe_a2a_initialize", &tensorrt_llm::torch_ext::moe_comm::moeA2AInitializeOp); module.impl("moe_a2a_sanitize_expert_ids", &tensorrt_llm::torch_ext::moe_comm::moeA2ASanitizeExpertIdsOp); + module.impl("moe_a2a_check_incomplete_round", &tensorrt_llm::torch_ext::moe_comm::moeA2ACheckIncompleteRoundOp); module.impl( "moe_a2a_get_combine_payload_tensor", &tensorrt_llm::torch_ext::moe_comm::moeA2AGetCombinePayloadTensorOp); } diff --git a/docs/source/developer-guide/moe-perf-guide.md b/docs/source/developer-guide/moe-perf-guide.md index a30d15d6724a..499371155e76 100644 --- a/docs/source/developer-guide/moe-perf-guide.md +++ b/docs/source/developer-guide/moe-perf-guide.md @@ -147,7 +147,7 @@ The value must be one of the strategy names in the table above. - `fence`: fence only. - `sentinel`: same as `auto`, but logs a warning naming the reason when sentinel cannot be used. -A step is carried by sentinel when its busiest rank sends at most `sentinel_max_tokens` tokens, it gathers no EPLB statistics, and its payload matches the layer's format (BF16, MXFP8 or NVFP4 rows). Sentinel is off for FP8 low-precision combine (`use_low_precision_moe_combine`), for communicators with the AlltoAll watchdog or EP-group health, for more than 64 EP ranks, and for the FP8 per-tensor, `W4A8_MXFP4_FP8` and `W4A8_NVFP4_FP8` activation formats. +A step is carried by sentinel when its busiest rank sends at most `sentinel_max_tokens` tokens, it gathers no EPLB statistics, and its payload matches the layer's format (BF16, MXFP8 or NVFP4 rows). Sentinel is off for FP8 low-precision combine (`use_low_precision_moe_combine`), for layers with Wide-EP fault tolerance (`enable_wide_ep_fault_tolerance`), for more than 64 EP ranks, and for the FP8 per-tensor, `W4A8_MXFP4_FP8` and `W4A8_NVFP4_FP8` activation formats. The default `sentinel_max_tokens` is between 159 and 512 tokens per rank (capped at `max_num_tokens_per_rank`); `TRTLLM_NVLINK_ONE_SIDED_A2A_SENTINEL_MAX_TOKENS` overrides it. The sentinel landing rows are added to the workspace, `ep_size × sentinel_max_tokens` of them per rank: for hidden 7168, top-8 BF16 rows this is about 52 / 105 / 210 / 419 MiB per rank at EP 8 / 16 / 32 / 64. If a `TRTLLM_NVLINK_ONE_SIDED_A2A_WORKSPACE_MB` override cannot hold them, sentinel is turned off with a warning. diff --git a/tensorrt_llm/_torch/alltoall_watchdog.py b/tensorrt_llm/_torch/alltoall_watchdog.py deleted file mode 100644 index 37a796cbbf01..000000000000 --- a/tensorrt_llm/_torch/alltoall_watchdog.py +++ /dev/null @@ -1,792 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Host-side watchdog for MoE AlltoAll completion flags. - -The NVLinkOneSided kernels signal each collective by writing the current -``flag_val`` into the rank-local completion flag table. A dead peer in the -silent-spin failure mode never writes its slot, so this watchdog polls the same -table from a CPU thread and reports peers whose flags do not reach the expected -generation before a bounded timeout. - -Timeouts are detection events, not committed membership changes. The optional -EP health object supplies the already-committed active peers to watch, and is -read-only here. Higher-layer recovery coordination consumes ``on_timeout`` -events and commits a new membership only after the data plane and expert -placement are ready for the same generation. -""" - -from __future__ import annotations - -import threading -import time -from collections import deque -from collections.abc import Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass -from typing import Protocol - -import torch - -from tensorrt_llm._utils import prefer_pinned -from tensorrt_llm.logger import logger as tllm_logger - -DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S = 5.0 -DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S = 0.1 -UNKNOWN_COMPLETION_FLAG = -(2**63) -_COMPLETION_FLAG_MASK = (1 << 32) - 1 -_COMPLETION_FLAG_HALF_RANGE = 1 << 31 -_ACTIVE_RANK_MASK_WORD_BITS = 64 -_ACTIVE_RANK_MASK_WORDS = 4 -_ACTIVE_RANK_MASK_WORD_MASK = (1 << _ACTIVE_RANK_MASK_WORD_BITS) - 1 -_WORKSPACE_WATCHDOG_STATE_KEY = "alltoall_watchdog_shared_state" -_WORKSPACE_WATCHDOG_STATE_INIT_LOCK = threading.Lock() - - -def reject_rank_mask_cuda_graph_capture(rank_mask_enabled: bool) -> None: - """Reject graph capture until membership-scoped recapture is available.""" - if rank_mask_enabled and torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "rank-mask mode does not support CUDA graphs until generation-scoped " - "invalidation and recapture are implemented" - ) - - -def _normalize_completion_flag(value: int) -> int: - return int(value) & _COMPLETION_FLAG_MASK - - -def _completion_flag_reached(observed: int, expected: int) -> bool: - if observed == UNKNOWN_COMPLETION_FLAG: - return False - # Counter values are ordered modulo uint32. A queued watch cannot lag by - # half the counter space within the bounded watchdog timeout. - distance = (_normalize_completion_flag(observed) - expected) & _COMPLETION_FLAG_MASK - return distance < _COMPLETION_FLAG_HALF_RANGE - - -class CompletionFlagReader(Protocol): - """Reads one phase's rank-local completion flag row.""" - - def read_completion_flags(self, phase: str) -> Sequence[int]: - """Return ``ep_size`` flag values for ``phase``.""" - - -class EPGroupHealthSnapshotLike(Protocol): - """Atomic fields required from a committed-membership snapshot.""" - - @property - def mask(self) -> int: - """Return the committed active-rank bitmask.""" - - @property - def generation(self) -> int: - """Return the matching committed-membership generation.""" - - -class EPGroupHealthLike(Protocol): - """Read-only committed EP membership used by AlltoAll frontends.""" - - def get_mask(self) -> int: - """Return the active-rank bitmask.""" - - def get_mask_words(self) -> tuple[int, ...]: - """Return the active-rank bitmask split into uint64 words.""" - - def snapshot(self) -> EPGroupHealthSnapshotLike: - """Return one atomic committed mask and generation snapshot.""" - - -@dataclass(frozen=True) -class ActiveRankMaskSnapshot: - """One dispatch epoch's rank mask and optional committed generation. - - ``committed_generation`` is populated only when ``active_rank_mask`` was - read from the coordinator-owned EP health object. Explicit caller masks do - not inherit a generation from unrelated health state. - """ - - active_rank_mask: torch.Tensor | None - committed_generation: int | None - - -class CompletionFlagReadTimeout(TimeoutError): - """Raised when the host watchdog cannot read completion flags in time.""" - - -@dataclass(frozen=True) -class AlltoAllWatchdogTimeout: - """Detection details emitted when an AlltoAll phase times out. - - ``missing_ranks`` contains suspects relative to the committed active mask - captured for this collective. Receiving this event does not publish a new - active mask. - """ - - phase: str - expected_flag: int - observed_flags: tuple[int, ...] - missing_ranks: tuple[int, ...] - elapsed_s: float - poll_timed_out: bool = False - - -@dataclass(frozen=True) -class _CollectiveWatch: - phase: str - expected_flag: int - active_mask: int - start_s: float - - -@dataclass(frozen=True) -class _SharedWatchdogConfig: - ep_size: int - ep_rank: int - timeout_s: float - poll_interval_s: float - health: EPGroupHealthLike | None - on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None - - -@dataclass -class _WorkspaceWatchdogState: - lock: threading.Lock - generation: int = 0 - watchdog: AlltoAllWatchdog | None = None - config: _SharedWatchdogConfig | None = None - ref_count: int = 0 - - -class _TorchCompletionFlagReader: - """Completion-flag reader backed by the MoE AlltoAll workspace tensor.""" - - def __init__( - self, - workspace: torch.Tensor, - ep_rank: int, - ep_size: int, - dispatch_completion_flags_offset: int, - combine_completion_flags_offset: int, - device_copy_timeout_s: float = DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - ) -> None: - if workspace.dim() != 2: - raise ValueError("workspace must be a 2D tensor [ep_size, size_per_rank]") - if not 0 <= ep_rank < ep_size: - raise ValueError(f"ep_rank must be in [0, {ep_size}), got {ep_rank}") - if workspace.size(0) != ep_size: - raise ValueError( - f"workspace first dimension must equal ep_size={ep_size}, got {workspace.size(0)}" - ) - self._workspace = workspace - self._ep_rank = ep_rank - self._ep_size = ep_size - self._offsets = { - "dispatch": int(dispatch_completion_flags_offset), - "combine": int(combine_completion_flags_offset), - } - self._device_copy_timeout_s = float(device_copy_timeout_s) - self._copy_stream: torch.cuda.Stream | None = None - self._host_flags: torch.Tensor | None = None - self._copy_event: torch.cuda.Event | None = None - self._retired_copies: list[tuple[torch.Tensor, torch.cuda.Event]] = [] - if workspace.device.type == "cuda": - self._copy_stream = torch.cuda.Stream(device=workspace.device) - - def _prune_retired_copies(self) -> None: - self._retired_copies = [ - (host_flags, event) for host_flags, event in self._retired_copies if not event.query() - ] - - def _read_cuda_flags(self, flags: torch.Tensor) -> tuple[int, ...]: - assert self._copy_stream is not None - self._prune_retired_copies() - - if self._host_flags is None: - self._host_flags = torch.empty( - (self._ep_size,), - dtype=torch.int32, - device="cpu", - pin_memory=prefer_pinned(), - ) - if self._copy_event is None: - self._copy_event = torch.cuda.Event(blocking=False) - host_flags = self._host_flags - event = self._copy_event - with torch.cuda.device(flags.device), torch.cuda.stream(self._copy_stream): - host_flags.copy_(flags.detach(), non_blocking=True) - event.record(self._copy_stream) - - deadline_s = time.monotonic() + self._device_copy_timeout_s - while not event.query(): - remaining_s = deadline_s - time.monotonic() - if remaining_s <= 0: - self._retired_copies.append((host_flags, event)) - self._host_flags = None - self._copy_event = None - raise CompletionFlagReadTimeout( - "timed out copying AlltoAll completion flags to host" - ) - time.sleep(min(remaining_s, 0.001)) - - return tuple(int(v) for v in host_flags.tolist()) - - def read_completion_flags(self, phase: str) -> tuple[int, ...]: - offset = self._offsets[phase] - end = offset + self._ep_size * 4 - flags = self._workspace[self._ep_rank, offset:end].view(torch.int32) - if flags.device.type == "cuda": - return self._read_cuda_flags(flags) - if flags.device.type != "cpu": - flags = flags.detach().cpu() - return tuple(int(v) for v in flags.tolist()) - - -class AlltoAllWatchdogCoordinator: - """Shared watchdog plumbing for MoE AlltoAll frontends. - - ``health`` is the committed membership snapshot source. This class reads - it when a dispatch starts but never publishes detected failures into it. - """ - - def __init__( - self, - *, - workspace_state: MutableMapping[str, object], - workspace: torch.Tensor, - metainfo: torch.Tensor, - metainfo_index: Mapping[str, int], - ep_rank: int, - health: EPGroupHealthLike | None = None, - ) -> None: - self._workspace_state = workspace_state - self._workspace = workspace - self._metainfo = metainfo - self._metainfo_index = metainfo_index - self._ep_rank = ep_rank - self._health = health - - def _find_shared_state(self) -> _WorkspaceWatchdogState | None: - state = self._workspace_state.get(_WORKSPACE_WATCHDOG_STATE_KEY) - if state is None: - return None - if not isinstance(state, _WorkspaceWatchdogState): - raise TypeError("invalid shared AlltoAll watchdog workspace state") - return state - - def _get_shared_state(self) -> _WorkspaceWatchdogState: - state = self._find_shared_state() - if state is not None: - return state - with _WORKSPACE_WATCHDOG_STATE_INIT_LOCK: - state = self._workspace_state.get(_WORKSPACE_WATCHDOG_STATE_KEY) - if state is None: - state = _WorkspaceWatchdogState(lock=threading.Lock()) - self._workspace_state[_WORKSPACE_WATCHDOG_STATE_KEY] = state - if not isinstance(state, _WorkspaceWatchdogState): - raise TypeError("invalid shared AlltoAll watchdog workspace state") - return state - - def read_current_flag_val(self) -> int: - flag_val_offset = int(self._metainfo[self._metainfo_index["FLAG_VAL_OFFSET_INDEX"]].item()) - flag_val = self._workspace[self._ep_rank, flag_val_offset : flag_val_offset + 4].view( - torch.int32 - ) - if flag_val.device.type != "cpu": - flag_val = flag_val.detach().cpu() - return _normalize_completion_flag(int(flag_val.item())) - - def capture_active_rank_mask( - self, active_rank_mask: torch.Tensor | None - ) -> ActiveRankMaskSnapshot: - """Capture a caller override or one coherent committed-mask epoch. - - The returned tensor does not alias a caller-owned override, so it can - safely represent the mask used by one dispatch/combine pair. When the - mask comes from ``health``, derive all fixed-width ABI words from one - atomic mask/generation snapshot. - """ - if active_rank_mask is not None: - return ActiveRankMaskSnapshot( - active_rank_mask=active_rank_mask.detach().clone(), - committed_generation=None, - ) - if self._health is None: - return ActiveRankMaskSnapshot( - active_rank_mask=None, - committed_generation=None, - ) - - health_snapshot = self._health.snapshot() - mask_words = tuple( - (health_snapshot.mask >> (_ACTIVE_RANK_MASK_WORD_BITS * index)) - & _ACTIVE_RANK_MASK_WORD_MASK - for index in range(_ACTIVE_RANK_MASK_WORDS) - ) - return ActiveRankMaskSnapshot( - active_rank_mask=torch.tensor( - mask_words, - dtype=torch.uint64, - device="cpu", - ), - committed_generation=health_snapshot.generation, - ) - - def active_rank_mask_tensor(self, active_rank_mask: torch.Tensor | None) -> torch.Tensor | None: - """Return only the tensor from :meth:`capture_active_rank_mask`. - - Dispatch frontends must retain the full snapshot so combine can verify - the committed generation before launching its kernel. - """ - return self.capture_active_rank_mask(active_rank_mask).active_rank_mask - - def active_rank_mask_for_combine( - self, - dispatch_snapshot: ActiveRankMaskSnapshot, - requested_active_rank_mask: torch.Tensor | None, - ) -> torch.Tensor | None: - """Validate the dispatch epoch and return its captured rank mask. - - A coordinator-owned generation change between dispatch and combine is - an invalid asynchronous commit, so fail closed before launching the - combine kernel. Explicit caller masks are independent of health and - therefore carry no committed generation. - """ - dispatch_active_rank_mask = dispatch_snapshot.active_rank_mask - committed_generation = dispatch_snapshot.committed_generation - if committed_generation is not None: - if self._health is None: - raise RuntimeError("committed rank-mask snapshot has no EP health source") - current_generation = self._health.snapshot().generation - if current_generation != committed_generation: - raise RuntimeError( - "committed EP membership changed between dispatch and combine " - f"(generation {committed_generation} -> {current_generation}); " - "aborting collective epoch" - ) - - if requested_active_rank_mask is None: - return dispatch_active_rank_mask - requested_mask = self.active_rank_mask_tensor(requested_active_rank_mask) - assert requested_mask is not None - if dispatch_active_rank_mask is None or not torch.equal( - dispatch_active_rank_mask, requested_mask - ): - raise ValueError("active_rank_mask must match the mask captured at dispatch") - return dispatch_active_rank_mask - - def active_mask_int(self, active_rank_mask: torch.Tensor | None) -> int | None: - if active_rank_mask is not None: - mask_cpu = active_rank_mask.detach().cpu() - return sum(int(word) << (64 * idx) for idx, word in enumerate(mask_cpu.tolist())) - if self._health is not None: - return self._health.get_mask() - return None - - def acquire_watchdog( - self, - *, - ep_size: int, - timeout_s: float, - poll_interval_s: float, - on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None = None, - ) -> AlltoAllWatchdog: - config = _SharedWatchdogConfig( - ep_size=int(ep_size), - ep_rank=self._ep_rank, - timeout_s=float(timeout_s), - poll_interval_s=float(poll_interval_s), - health=self._health, - on_timeout=on_timeout, - ) - state = self._get_shared_state() - with state.lock: - if state.watchdog is None: - state.generation = self.read_current_flag_val() - state.watchdog = AlltoAllWatchdog.from_workspace( - workspace=self._workspace, - metainfo=self._metainfo, - metainfo_index=self._metainfo_index, - ep_rank=self._ep_rank, - ep_size=ep_size, - timeout_s=timeout_s, - poll_interval_s=poll_interval_s, - health=self._health, - on_timeout=on_timeout, - ) - state.config = config - elif not self._matching_config(state.config, config): - raise ValueError( - "AlltoAll wrappers sharing a workspace must use the same " - "watchdog configuration and EP health object" - ) - state.ref_count += 1 - watchdog = state.watchdog - assert watchdog is not None - return watchdog - - @staticmethod - def _matching_config( - existing: _SharedWatchdogConfig | None, - requested: _SharedWatchdogConfig, - ) -> bool: - return ( - existing is not None - and existing.ep_size == requested.ep_size - and existing.ep_rank == requested.ep_rank - and existing.timeout_s == requested.timeout_s - and existing.poll_interval_s == requested.poll_interval_s - and existing.health is requested.health - and existing.on_timeout is requested.on_timeout - ) - - def release_watchdog(self, watchdog: AlltoAllWatchdog) -> None: - state = self._get_shared_state() - watchdog_to_stop: AlltoAllWatchdog | None = None - with state.lock: - if state.watchdog is not watchdog or state.ref_count <= 0: - raise RuntimeError("attempted to release an unregistered AlltoAll watchdog") - state.ref_count -= 1 - if state.ref_count == 0: - watchdog_to_stop = state.watchdog - state.watchdog = None - state.config = None - if watchdog_to_stop is not None: - watchdog_to_stop.stop() - - def watch_collective( - self, - watchdog: AlltoAllWatchdog | None, - phase: str, - active_rank_mask: torch.Tensor | None, - ) -> None: - if watchdog is None: - state = self._find_shared_state() - if state is not None: - with state.lock: - if state.watchdog is not None: - state.generation = (state.generation + 1) & _COMPLETION_FLAG_MASK - return - active_mask = self.active_mask_int(active_rank_mask) - state = self._get_shared_state() - with state.lock: - if state.watchdog is not watchdog: - raise RuntimeError("AlltoAll watchdog is not registered for this workspace") - state.generation = (state.generation + 1) & _COMPLETION_FLAG_MASK - watchdog.watch( - phase=phase, - expected_flag=state.generation, - active_mask=active_mask, - ) - - -class AlltoAllWatchdog: - """Background host thread that watches AlltoAll completion flags. - - The watchdog is intentionally opt-in. Callers queue phases with - :meth:`watch`; the thread polls them in FIFO order so a queued combine cannot - hide a still-spinning dispatch. A timeout is reported through - ``on_timeout`` without mutating the committed EP membership. - """ - - VALID_PHASES = frozenset({"dispatch", "combine"}) - - def __init__( - self, - *, - ep_size: int, - ep_rank: int, - completion_reader: CompletionFlagReader, - timeout_s: float = DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, - poll_interval_s: float = DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - health: EPGroupHealthLike | None = None, - on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None = None, - ) -> None: - if ep_size <= 0: - raise ValueError(f"ep_size must be > 0, got {ep_size}") - if not 0 <= ep_rank < ep_size: - raise ValueError(f"ep_rank must be in [0, {ep_size}), got {ep_rank}") - if timeout_s <= 0: - raise ValueError(f"timeout_s must be > 0, got {timeout_s}") - if poll_interval_s <= 0: - raise ValueError(f"poll_interval_s must be > 0, got {poll_interval_s}") - - self._ep_size = int(ep_size) - self._ep_rank = int(ep_rank) - self._completion_reader = completion_reader - self._timeout_s = float(timeout_s) - self._poll_interval_s = float(poll_interval_s) - self._health = health - self._on_timeout = on_timeout - - self._cv = threading.Condition() - self._queue: deque[_CollectiveWatch] = deque() - self._closed = False - self._stopping = False - self._thread: threading.Thread | None = None - self._last_error: BaseException | None = None - - @classmethod - def from_workspace( - cls, - *, - workspace: torch.Tensor, - metainfo: torch.Tensor, - metainfo_index: Mapping[str, int], - ep_rank: int, - ep_size: int, - timeout_s: float = DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, - poll_interval_s: float = DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - health: EPGroupHealthLike | None = None, - on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None = None, - ) -> "AlltoAllWatchdog": - """Build a watchdog from the MoE AlltoAll workspace and metainfo.""" - dispatch_offset = int( - metainfo[metainfo_index["DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX"]].item() - ) - combine_offset = int( - metainfo[metainfo_index["COMBINE_COMPLETION_FLAGS_OFFSET_INDEX"]].item() - ) - reader = _TorchCompletionFlagReader( - workspace=workspace, - ep_rank=ep_rank, - ep_size=ep_size, - dispatch_completion_flags_offset=dispatch_offset, - combine_completion_flags_offset=combine_offset, - device_copy_timeout_s=poll_interval_s, - ) - return cls( - ep_size=ep_size, - ep_rank=ep_rank, - completion_reader=reader, - timeout_s=timeout_s, - poll_interval_s=poll_interval_s, - health=health, - on_timeout=on_timeout, - ) - - @property - def last_error(self) -> BaseException | None: - """Return the last polling-thread error, if any.""" - with self._cv: - return self._last_error - - def start(self) -> None: - """Start the background polling thread. Idempotent.""" - with self._cv: - if self._closed: - raise RuntimeError("cannot start a stopped AlltoAllWatchdog") - if self._thread is not None and self._thread.is_alive(): - return - self._stopping = False - self._thread = threading.Thread( - target=self._run, - name=f"AlltoAllWatchdog-rank{self._ep_rank}", - daemon=True, - ) - self._thread.start() - - def stop(self, timeout_s: float | None = None) -> None: - """Stop the polling thread and wait for it to exit.""" - with self._cv: - self._closed = True - self._stopping = True - self._queue.clear() - self._cv.notify_all() - thread = self._thread - if thread is not None and thread is not threading.current_thread(): - thread.join(timeout=timeout_s) - - def watch( - self, - *, - phase: str, - expected_flag: int, - active_mask: int | None = None, - ) -> None: - """Queue a just-launched AlltoAll phase for watchdog polling.""" - if phase not in self.VALID_PHASES: - raise ValueError(f"phase must be one of {sorted(self.VALID_PHASES)}, got {phase!r}") - if not 0 <= expected_flag <= _COMPLETION_FLAG_MASK: - raise ValueError( - f"expected_flag must be in [0, {_COMPLETION_FLAG_MASK}], got {expected_flag}" - ) - if active_mask is None: - if self._health is not None: - active_mask = self._health.get_mask() - else: - active_mask = (1 << self._ep_size) - 1 - if not (active_mask >> self._ep_rank) & 1: - raise ValueError("active_mask must include the local ep_rank") - - self.start() - with self._cv: - if self._closed: - raise RuntimeError("cannot queue a stopped AlltoAllWatchdog") - self._queue.append( - _CollectiveWatch( - phase=phase, - expected_flag=int(expected_flag), - active_mask=int(active_mask), - start_s=time.monotonic(), - ) - ) - self._cv.notify_all() - - def wait_until_idle(self, timeout_s: float) -> bool: - """Wait until all queued phases complete or timeout handling clears them.""" - deadline = time.monotonic() + timeout_s - with self._cv: - while self._queue: - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._cv.wait(timeout=remaining) - return True - - def __enter__(self) -> "AlltoAllWatchdog": - self.start() - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.stop(timeout_s=1.0) - - def _active_ranks(self, active_mask: int) -> tuple[int, ...]: - return tuple(rank for rank in range(self._ep_size) if (active_mask >> rank) & 1) - - def _phase_complete(self, watch: _CollectiveWatch, observed_flags: tuple[int, ...]) -> bool: - return all( - _completion_flag_reached(observed_flags[rank], watch.expected_flag) - for rank in self._active_ranks(watch.active_mask) - ) - - def _missing_ranks( - self, watch: _CollectiveWatch, observed_flags: tuple[int, ...] - ) -> tuple[int, ...]: - return tuple( - rank - for rank in self._active_ranks(watch.active_mask) - if not _completion_flag_reached(observed_flags[rank], watch.expected_flag) - ) - - def _handle_timeout( - self, - watch: _CollectiveWatch, - observed_flags: tuple[int, ...], - *, - poll_timed_out: bool = False, - ) -> None: - elapsed_s = time.monotonic() - watch.start_s - missing_ranks = self._missing_ranks(watch, observed_flags) - event = AlltoAllWatchdogTimeout( - phase=watch.phase, - expected_flag=watch.expected_flag, - observed_flags=observed_flags, - missing_ranks=missing_ranks, - elapsed_s=elapsed_s, - poll_timed_out=poll_timed_out, - ) - if poll_timed_out: - tllm_logger.error( - "AlltoAll watchdog could not read completion flags on rank %d " - "during %s before timeout %.3fs; expected flag %d, active " - "ranks %s, observed flags %s; reporting detection event only", - self._ep_rank, - watch.phase, - elapsed_s, - watch.expected_flag, - list(self._active_ranks(watch.active_mask)), - list(observed_flags), - ) - else: - tllm_logger.warning( - "AlltoAll watchdog timeout on rank %d during %s: expected flag %d, " - "missing ranks %s, observed flags %s", - self._ep_rank, - watch.phase, - watch.expected_flag, - list(missing_ranks), - list(observed_flags), - ) - if self._on_timeout is not None: - self._on_timeout(event) - - def _stop_after_error(self, exc: Exception) -> None: - with self._cv: - self._last_error = exc - self._closed = True - self._stopping = True - self._queue.clear() - self._cv.notify_all() - - def _run(self) -> None: - last_observed_flags = tuple(UNKNOWN_COMPLETION_FLAG for _ in range(self._ep_size)) - poll_timed_out = False - while True: - with self._cv: - while not self._queue and not self._stopping: - self._cv.wait() - if self._stopping: - return - watch = self._queue[0] - - try: - observed_flags = tuple( - _normalize_completion_flag(int(v)) - for v in self._completion_reader.read_completion_flags(watch.phase) - ) - if len(observed_flags) != self._ep_size: - raise RuntimeError( - f"completion reader returned {len(observed_flags)} flags; " - f"expected ep_size={self._ep_size}" - ) - last_observed_flags = observed_flags - poll_timed_out = False - except CompletionFlagReadTimeout: - observed_flags = last_observed_flags - poll_timed_out = True - except Exception as exc: # noqa: BLE001 - keep watchdog failures visible. - self._stop_after_error(exc) - tllm_logger.error("AlltoAll watchdog stopped after polling error: %s", exc) - return - - if self._phase_complete(watch, observed_flags): - with self._cv: - if self._queue and self._queue[0] is watch: - self._queue.popleft() - self._cv.notify_all() - last_observed_flags = tuple(UNKNOWN_COMPLETION_FLAG for _ in range(self._ep_size)) - poll_timed_out = False - continue - - if time.monotonic() - watch.start_s >= self._timeout_s: - try: - self._handle_timeout(watch, observed_flags, poll_timed_out=poll_timed_out) - except Exception as exc: # noqa: BLE001 - keep watchdog failures visible. - self._stop_after_error(exc) - tllm_logger.error( - "AlltoAll watchdog stopped after timeout handling error: %s", exc - ) - return - with self._cv: - # The GPU stream is no longer trustworthy once a collective - # times out. Drop queued follow-on phases so they do not - # produce duplicate or misleading reports. - self._queue.clear() - self._cv.notify_all() - last_observed_flags = tuple(UNKNOWN_COMPLETION_FLAG for _ in range(self._ep_size)) - poll_timed_out = False - continue - - with self._cv: - self._cv.wait(timeout=self._poll_interval_s) diff --git a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py index a5a9df3b1ece..852b20873544 100644 --- a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py @@ -872,6 +872,7 @@ def _( enable_rank_mask: bool = False, active_rank_mask: Optional[torch.Tensor] = None, use_sentinel: bool = False, + abort_state: Optional[torch.Tensor] = None, ) -> Tuple[List[torch.Tensor], int, torch.Tensor]: recv_tensors: List[torch.Tensor] = [] for payload in input_payloads: @@ -905,6 +906,7 @@ def _( enable_rank_mask: bool = False, active_rank_mask: Optional[torch.Tensor] = None, use_sentinel: bool = False, + abort_state: Optional[torch.Tensor] = None, ) -> torch.Tensor: return payload.new_empty((local_num_tokens, payload.shape[2])) @@ -914,6 +916,7 @@ def _( metainfo: torch.Tensor, ep_rank: int, ep_size: int, + barrier: bool = True, ) -> None: return None @@ -927,6 +930,15 @@ def _( ) -> None: return None + @torch.library.register_fake("trtllm::moe_a2a_check_incomplete_round") + def _( + workspace: torch.Tensor, + metainfo: torch.Tensor, + ep_rank: int, + abort_state: torch.Tensor, + ) -> None: + return None + @torch.library.register_fake("trtllm::moe_a2a_get_combine_payload_tensor") def _( workspace: torch.Tensor, diff --git a/tensorrt_llm/_torch/mnnvl_alltoall_workspace.py b/tensorrt_llm/_torch/mnnvl_alltoall_workspace.py index 7238597c5b08..4cb3ab93ac48 100644 --- a/tensorrt_llm/_torch/mnnvl_alltoall_workspace.py +++ b/tensorrt_llm/_torch/mnnvl_alltoall_workspace.py @@ -13,18 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass -from typing import Callable, Mapping, MutableMapping, Protocol, cast +from typing import Callable, MutableMapping, Protocol, cast from weakref import WeakSet import torch -from tensorrt_llm._torch.alltoall_watchdog import ( - AlltoAllWatchdog, - AlltoAllWatchdogCoordinator, - AlltoAllWatchdogTimeout, - EPGroupHealthLike, -) from tensorrt_llm._torch.distributed.mnnvl_memory import ( MnnvlCheckpointCommunicator, MnnvlMemory, @@ -83,16 +76,8 @@ def _mnnvl_checkpoint_reset(self) -> None: """Reset frontend protocol state after a successful restore.""" -@dataclass(frozen=True) -class _WatchdogConfig: - ep_size: int - timeout_s: float - poll_interval_s: float - on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None - - class _MnnvlAlltoAllWorkspaceLifecycle: - """Own checkpoint and watchdog transitions for one shared MoE workspace. + """Own checkpoint transitions for one shared MoE workspace. ``PyExecutor`` invokes this resource hook from the engine sleep/wakeup PREPARE/COMMIT/ABORT control path. That coordinator stops admission, drains @@ -109,24 +94,16 @@ def __init__( memory: MnnvlMemory, workspace: torch.Tensor, metainfo: torch.Tensor, - metainfo_index: Mapping[str, int], ep_rank: int, ep_size: int, - health: EPGroupHealthLike | None, ) -> None: self._workspace_state = workspace_state self._memory = memory self._workspace = workspace self._metainfo = metainfo - self._metainfo_index = dict(metainfo_index) self._ep_rank = int(ep_rank) self._ep_size = int(ep_size) - self._health = health self._clients: WeakSet[_WorkspaceClient] = WeakSet() - self._watchdog_clients: WeakSet[_WorkspaceClient] = WeakSet() - self._watchdog_config: _WatchdogConfig | None = None - self._watchdog: AlltoAllWatchdog | None = None - self._coordinator = self._create_coordinator() @classmethod def get_or_create( @@ -136,10 +113,8 @@ def get_or_create( memory: MnnvlMemory, workspace: torch.Tensor, metainfo: torch.Tensor, - metainfo_index: Mapping[str, int], ep_rank: int, ep_size: int, - health: EPGroupHealthLike | None, ) -> "_MnnvlAlltoAllWorkspaceLifecycle": lifecycle = workspace_state.get(_WORKSPACE_LIFECYCLE_KEY) if lifecycle is None: @@ -148,10 +123,8 @@ def get_or_create( memory=memory, workspace=workspace, metainfo=metainfo, - metainfo_index=metainfo_index, ep_rank=ep_rank, ep_size=ep_size, - health=health, ) workspace_state[_WORKSPACE_LIFECYCLE_KEY] = lifecycle return lifecycle @@ -161,10 +134,8 @@ def get_or_create( memory=memory, workspace=workspace, metainfo=metainfo, - metainfo_index=metainfo_index, ep_rank=ep_rank, ep_size=ep_size, - health=health, ) return lifecycle @@ -172,59 +143,15 @@ def get_or_create( def metainfo(self) -> torch.Tensor: return self._metainfo - @property - def coordinator(self) -> AlltoAllWatchdogCoordinator: - return self._coordinator - - def watchdog_for(self, client: _WorkspaceClient) -> AlltoAllWatchdog | None: - if client not in self._watchdog_clients: - return None - return self._watchdog - - def register( - self, - client: _WorkspaceClient, - *, - watchdog_timeout_s: float | None, - watchdog_poll_interval_s: float, - watchdog_on_timeout: Callable[[AlltoAllWatchdogTimeout], None] | None, - ) -> None: - if client in self._clients: - return - if watchdog_timeout_s is None: - self._clients.add(client) - return - - config = _WatchdogConfig( - ep_size=self._ep_size, - timeout_s=float(watchdog_timeout_s), - poll_interval_s=float(watchdog_poll_interval_s), - on_timeout=watchdog_on_timeout, - ) - self._validate_watchdog_config(config) + def register(self, client: _WorkspaceClient) -> None: self._clients.add(client) - self._watchdog_clients.add(client) - try: - if self._memory.mapped: - self._start_watchdog() - except Exception: - self._watchdog_clients.discard(client) - self._clients.discard(client) - if not self._watchdog_clients: - self._watchdog_config = None - raise def unregister(self, client: _WorkspaceClient) -> None: self._clients.discard(client) - self._watchdog_clients.discard(client) - if not self._watchdog_clients: - self._stop_watchdog() - self._watchdog_config = None def checkpoint_prepare(self) -> None: """Preflight shared readers, then collectively detach backing handles.""" if not self._memory.mapped: - self._stop_watchdog() self._memory.checkpoint_prepare() return local_clients_idle = all( @@ -241,13 +168,11 @@ def checkpoint_prepare(self) -> None: ) except TimeoutError: self._memory.checkpoint_fail_closed() - self._stop_watchdog() raise if active_ranks: raise RuntimeError( f"Cannot checkpoint during an active MoE All-to-All phase on ranks {active_ranks}" ) - self._stop_watchdog() self._memory.checkpoint_prepare() def checkpoint_restore( @@ -272,9 +197,7 @@ def checkpoint_restore( ) self._metainfo = refreshed_metainfo self._workspace_state["metainfo"] = refreshed_metainfo - self._coordinator = self._create_coordinator() torch.cuda.synchronize() - self._start_watchdog() for client in list(self._clients): client._mnnvl_checkpoint_reset() except Exception as error: @@ -286,7 +209,6 @@ def checkpoint_restore( expected_size=self._ep_size, ) if unready_ranks: - self._stop_watchdog() if local_error is not None: raise local_error raise RuntimeError( @@ -296,52 +218,21 @@ def checkpoint_restore( self._memory._checkpoint_restore_complete() except Exception: self._memory._checkpoint_restore_failed() - self._stop_watchdog() raise - def _create_coordinator(self) -> AlltoAllWatchdogCoordinator: - return AlltoAllWatchdogCoordinator( - workspace_state=self._workspace_state, - workspace=self._workspace, - metainfo=self._metainfo, - metainfo_index=self._metainfo_index, - ep_rank=self._ep_rank, - health=self._health, - ) - - def _start_watchdog(self) -> None: - config = self._watchdog_config - if config is None or not self._watchdog_clients or self._watchdog is not None: - return - self._watchdog = self._coordinator.acquire_watchdog( - ep_size=config.ep_size, - timeout_s=config.timeout_s, - poll_interval_s=config.poll_interval_s, - on_timeout=config.on_timeout, - ) - - def _stop_watchdog(self) -> None: - if self._watchdog is None: - return - self._coordinator.release_watchdog(self._watchdog) - self._watchdog = None - def _validate_shared_context( self, *, memory: MnnvlMemory, workspace: torch.Tensor, metainfo: torch.Tensor, - metainfo_index: Mapping[str, int], ep_rank: int, ep_size: int, - health: EPGroupHealthLike | None, ) -> None: if ( self._memory is not memory or self._workspace is not workspace or self._metainfo is not metainfo - or self._metainfo_index != dict(metainfo_index) or self._ep_rank != ep_rank or self._ep_size != ep_size ): @@ -349,27 +240,3 @@ def _validate_shared_context( "MNNVL All-to-All wrappers sharing a workspace must use the " "same allocation, metadata layout, and rank layout" ) - if self._health is health: - return - if self._clients or self._watchdog is not None or self._watchdog_config is not None: - raise ValueError( - "MNNVL All-to-All wrappers sharing a workspace must use the same EP health object" - ) - self._health = health - self._coordinator = self._create_coordinator() - - def _validate_watchdog_config(self, requested: _WatchdogConfig) -> None: - existing = self._watchdog_config - if existing is None: - self._watchdog_config = requested - return - if ( - existing.ep_size != requested.ep_size - or existing.timeout_s != requested.timeout_s - or existing.poll_interval_s != requested.poll_interval_s - or existing.on_timeout is not requested.on_timeout - ): - raise ValueError( - "MNNVL All-to-All wrappers sharing a workspace must use the " - "same watchdog configuration" - ) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index a89fa5451bb7..83b42995a616 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -428,6 +428,8 @@ class ModelConfig(Generic[TConfig]): megamoe_fuse_shared_expert: bool = False # If true, use low precision combine in MoE operations (only for NVFP4 quantization) use_low_precision_moe_combine: bool = False + # If true, the MoE all-to-all runs in its Wide-EP fault-tolerant mode + enable_wide_ep_fault_tolerance: bool = False # NVFP4 GEMM backend configuration - list of backends to consider for auto-selection nvfp4_gemm_allowed_backends: List[str] = field( diff --git a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md index 15be89eca904..0055a0447378 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -251,6 +251,10 @@ strategy checks or private workspace attributes. The executor request queue owns admission around the checkpoint and reopens it only after a complete wakeup. +### Wide-EP fault tolerance + +Under `enable_wide_ep_fault_tolerance`, `CommunicationFactory` builds `NVLinkOneSided` with its rank-mask kernels, and the model loader binds every MoE layer's communication to one `MoeExecutionState` (`communication/execution_state.py`, bound by `pyexecutor/rank_failure_model_load.py`) before the weights load. The kernels exchange only with the EP ranks set in the state's device mask, which they read when they run, so captured CUDA graphs follow a mask that `install_mask` rewrites while no collective runs. They stop waiting on peers once the state's pinned host abort state is requested (`request_abort`), and a wait on a peer longer than about 10 ms publishes the round flag it waits for there (`waited_flags`). A collective that ends without some active peer marks its round incomplete in the workspace, so later collectives exchange with this rank only until `rearm` resets the workspaces. These layers set up no sentinel transport, whose kernels take no rank mask. + ### Core Design Principles 1. **Composition over inheritance** — Backend, Communication, EPLB, and Scheduler are independent, composable components @@ -302,7 +306,7 @@ with their ids, families and priorities, are the catalog generated into | Building and running a layer | `create_moe.py` builds the layer once `moe_resolution` has named the class; `configurable_moe.py` is the orchestrator that wires backend, communication, EPLB and scheduler and owns `forward_impl`; `moe_scheduler.py` holds the forward-execution strategies | | What implementations build on | `interface.py` (the complete-layer base `MoE`, `MoEWeightLoadingMode`, `MoESchedulerKind`), `impl_base.py` (the execution unit `MoEImplBase`, and `apply_moe_impl_construction_state()`, which every execution unit must call), `impl_blocks.py` (`MoEExecutionContractMixin`, and `MoEWeightOwnerMixin` with the `_weights_created` / `_weights_transformed` guards declared before any weight load), `weight_owner.py` (whether a module owns expert weights), `activation.py` (the activation vocabulary, and the only place a semantic constant becomes a kernel register), `quantization.py` (the per-format weight methods), `routing.py` | | Implementations | One package per family, laid out as in [Inside a family package](#inside-a-family-package): `cutlass/`, `trtllm_gen/`, `cutedsl/`, `marlin/`, `mega_moe/`. The rest are modules beside them: `fused_moe_deepgemm.py`, `fused_moe_cute_dsl_b12x.py` (both B12x leaves), and the legacy standalone layers `fused_moe_triton.py` and `fused_moe_vanilla.py`. A `fused_moe_.py` module above a package holds only the family's name and re-exports. What each family is and how it is built: [Implementation Families](#implementation-families) | -| Communication | `communication/`: the strategies, the `Communication` ABC in `base.py`, and `communication_factory.py`, which picks a strategy at runtime from hardware and configuration (`FUSED_COMM` backends skip it); see [Communication checkpointing](#communication-checkpointing). Beside it, the expert-parallel helpers (`deep_ep_utils.py`, `nccl_ep_utils.py`, `ep_group_health.py`, `wide_ep_ft.py`) and `moe_comm_timeout_guard.py` (see [Communication timeouts](#communication-timeouts)) | +| Communication | `communication/`: the strategies, the `Communication` ABC in `base.py`, and `communication_factory.py`, which picks a strategy at runtime from hardware and configuration (`FUSED_COMM` backends skip it); see [Communication checkpointing](#communication-checkpointing). Beside it, the expert-parallel helpers (`deep_ep_utils.py`, `nccl_ep_utils.py`, `ep_group_health.py`) and `moe_comm_timeout_guard.py` (see [Communication timeouts](#communication-timeouts)) | | EPLB | `moe_load_balancer.py`; per-layer dynamic EPLB in `per_layer_dynamic_eplb/`, created by `maybe_create_moe_load_balancer`: the model-level `manager`, the per-layer handle `layer`, the live-weight `arena` and the HALO-Q/TMA `planner` | | Python-side kernels and ops | `tensorrt_llm/_torch/moe/kernels/` (CuTe DSL kernels by architecture, and the MegaMoE sources) and `tensorrt_llm/_torch/moe/custom_ops/`; the MoE ops among `tensorrt_llm/_torch/custom_ops/`; the Triton kernels some leaves call, at the top of this directory (`fused_moe_triton_fp8_block_scale.py`, `triton_dequant_nvfp4.py`) | | C++ kernels, ops and gtests | `cpp/tensorrt_llm/kernels/moe/`, `cpp/tensorrt_llm/thop/moe/`, `cpp/tests/unit_tests/kernels/moe/`; see [C++](#c-cpp) | diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py b/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py index ba04937be454..9a9476b25df0 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py @@ -34,7 +34,6 @@ nccl_ep_supports_group_timeout, nccl_ep_supports_version, ) -from ..wide_ep_ft import get_wide_ep_ft_options from .allgather_reducescatter import AllGatherReduceScatter from .base import Communication from .deep_ep import DeepEP @@ -220,9 +219,6 @@ def create_strategy( try: enable_eplb = model_config.moe_load_balancer is not None - ep_group_health, watchdog_timeout_s, watchdog_poll_interval_s = get_wide_ep_ft_options( - model_config - ) strategy = NVLinkOneSided( mapping, num_slots, @@ -233,9 +229,7 @@ def create_strategy( dtype=act_dtype, num_experts=num_experts if enable_eplb else None, use_low_precision_combine=use_low_precision_combine, - ep_group_health=ep_group_health, - alltoall_watchdog_timeout_s=watchdog_timeout_s, - alltoall_watchdog_poll_interval_s=watchdog_poll_interval_s, + enable_wide_ep_fault_tolerance=model_config.enable_wide_ep_fault_tolerance, sentinel_payload_format=sentinel_format, ) logger.info("Selected communication strategy: NVLinkOneSided") @@ -245,6 +239,11 @@ def create_strategy( # selecting another strategy. raise except Exception as e: + if model_config.enable_wide_ep_fault_tolerance: + raise ValueError( + "enable_wide_ep_fault_tolerance requires the NVLinkOneSided all-to-all, " + f"which is unavailable: {e}" + ) from e logger.info(f"NVLinkOneSided not available: {e}") # Non-divisible EP: NVLinkTwoSided and DeepEP require num_experts % ep_size == 0. @@ -409,9 +408,6 @@ def _create_forced_method( ) elif method in ["NVLINK_ONE_SIDED"]: enable_eplb = model_config.moe_load_balancer is not None - ep_group_health, watchdog_timeout_s, watchdog_poll_interval_s = get_wide_ep_ft_options( - model_config - ) return NVLinkOneSided( mapping, num_slots, @@ -422,9 +418,7 @@ def _create_forced_method( dtype=act_dtype, num_experts=num_experts if enable_eplb else None, use_low_precision_combine=use_low_precision_combine, - ep_group_health=ep_group_health, - alltoall_watchdog_timeout_s=watchdog_timeout_s, - alltoall_watchdog_poll_interval_s=watchdog_poll_interval_s, + enable_wide_ep_fault_tolerance=model_config.enable_wide_ep_fault_tolerance, sentinel_payload_format=sentinel_format, ) elif method == "DEEPEP": diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py b/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py new file mode 100644 index 000000000000..1021ee996069 --- /dev/null +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The active-rank mask and abort state that the MoE all-to-all kernels read under Wide-EP fault tolerance. + +When a peer stops, ``request_abort`` releases the collectives that wait for it. Then, with the model +idle, ``rearm`` returns the bound workspaces and layers to their first round, and ``install_mask`` +writes the new rank mask. Long waits on peers publish the round flag they wait for in the abort +state (``waited_flags``). After each forward, ``check_incomplete_rounds`` marks the abort state +cancelled if a peer may have overwritten rows that the forward's combines read. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import torch + +from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm.bindings import internal as _tllm_internal + +from ..ep_group_health import EP_MASK_NUM_WORDS, EPGroupHealth + +_MASK_WORD_BITS = 64 +_ABORT_REQUESTED = 0 +# The words ``requested`` and ``status``, then the round flag awaited from each peer. +_ABORT_WORDS = slice(0, int(_tllm_internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX)) +_WAITED_FLAGS = slice(int(_tllm_internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX), None) + + +def _mask_words(mask: int, ep_rank: int) -> tuple[int, ...]: + """Split ``mask`` into the uint64 words of the kernels' mask, which must mark ``ep_rank``.""" + word_mask = (1 << _MASK_WORD_BITS) - 1 + words = tuple((mask >> (i * _MASK_WORD_BITS)) & word_mask for i in range(EP_MASK_NUM_WORDS)) + if not (words[ep_rank // _MASK_WORD_BITS] >> (ep_rank % _MASK_WORD_BITS)) & 1: + raise RuntimeError(f"active_rank_mask must mark the local ep_rank ({ep_rank}) as active") + return words + + +class MoeExecutionState: + """Own the active-rank mask and abort state that the kernels of a model's NVLinkOneSided layers read. + + Both stay at one address and change only in place, so CUDA graphs that captured the all-to-all + kernels follow every mask that :meth:`install_mask` writes and every abort request. + + Args: + ep_group_health: The EP membership whose active ranks the mask holds. + ep_rank: This rank's EP rank, which every mask must mark active. + device: The device of the all-to-all workspace. + """ + + def __init__(self, ep_group_health: EPGroupHealth, ep_rank: int, device: torch.device) -> None: + self.ep_group_health = ep_group_health + self.ep_rank = ep_rank + self._device = device + # The kernels read the abort state at its host address, which must be pinned memory. + self.abort = torch.zeros( + int(_tllm_internal.thop.MOE_A2A_ABORT_STATE_WORDS), + dtype=torch.int32, + pin_memory=prefer_pinned(), + ) + if not self.abort.is_pinned(): + raise RuntimeError("The MoE All-to-All abort state requires pinned host memory") + self._abort_words = self.abort.numpy()[_ABORT_WORDS] + self._workspaces: dict[object, tuple[Callable[[], None], Callable[[], None]]] = {} + self._layer_resets: list[Callable[[], None]] = [] + snapshot = ep_group_health.snapshot() + self._generation = snapshot.generation + self.active_rank_mask = torch.tensor( + _mask_words(snapshot.mask, ep_rank), dtype=torch.uint64, device=device + ) + + def register_layer(self, reset_state: Callable[[], None]) -> None: + """Register the reset of a bound layer's dispatch/combine state machine.""" + self._layer_resets.append(reset_state) + + def register_workspace( + self, + workspace: object, + reset_workspace: Callable[[], None], + check_incomplete_round: Callable[[], None], + ) -> None: + """Register a bound layer's workspace, once per workspace that layers share. + + ``reset_workspace`` resets this rank's row of it, and ``check_incomplete_round`` enqueues the + check of this rank's incomplete-round word in it (:meth:`check_incomplete_rounds`). + """ + self._workspaces.setdefault(workspace, (reset_workspace, check_incomplete_round)) + + def request_abort(self) -> None: + """Make running and future collectives return without their peers. Safe from any thread.""" + self._abort_words[_ABORT_REQUESTED] = 1 + + def aborted(self) -> bool: + """Return whether an abort was requested or a collective aborted since the last :meth:`rearm`.""" + return bool(self._abort_words.any()) + + def check_incomplete_rounds(self) -> None: + """Enqueue the check of this rank's incomplete-round word in every bound workspace. + + A peer sets the word before it overwrites rows that this rank's combines may have read, and + the check then marks the abort state cancelled, so :meth:`aborted` holds. Call once per + forward, on the stream of its last combine, before the event that the host waits on to + accept the forward. + """ + for _, check_incomplete_round in self._workspaces.values(): + check_incomplete_round() + + def waited_flags(self) -> dict[int, int]: + """Return each EP rank that a kernel has long waited on, with the round flag it waits for.""" + # The abort state stores the uint32 flags in int32 words. + return { + peer: round_flag & 0xFFFFFFFF + for peer, round_flag in enumerate(self.abort[_WAITED_FLAGS].tolist()) + if round_flag + } + + def rearm(self) -> None: + """Clear the abort state and return every bound workspace and layer to its first round. + + Call only while the model is idle and before :meth:`install_mask`. The device is + synchronized before and after, so no queued collective sees a partial reset. + """ + torch.cuda.synchronize(self._device) + self.abort.zero_() + for reset_workspace, _ in self._workspaces.values(): + reset_workspace() + for reset_state in self._layer_resets: + reset_state() + torch.cuda.synchronize(self._device) + + def install_mask(self) -> None: + """Write the active ranks of the EP group health into the device mask. + + Call only while no all-to-all collective is queued or running, and after :meth:`rearm` if a + collective aborted. The device is synchronized first, so no collective reads a partly + written mask. + """ + snapshot = self.ep_group_health.snapshot() + if snapshot.generation == self._generation: + return + words = _mask_words(snapshot.mask, self.ep_rank) + torch.cuda.synchronize(self._device) + if self.aborted(): + raise RuntimeError(f"MoE All-to-All abort state is {self.abort[_ABORT_WORDS].tolist()}") + self.active_rank_mask.copy_(torch.tensor(words, dtype=torch.uint64)) + self._generation = snapshot.generation diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py index fec980e31cc3..b7eaaa84c253 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py @@ -26,20 +26,10 @@ import os import sys -from typing import Callable, Dict, List, NamedTuple, Optional, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import torch -from tensorrt_llm._torch.alltoall_watchdog import ( - DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, - ActiveRankMaskSnapshot, - AlltoAllWatchdog, - AlltoAllWatchdogCoordinator, - AlltoAllWatchdogTimeout, - EPGroupHealthLike, - reject_rank_mask_cuda_graph_capture, -) from tensorrt_llm._torch.distributed.mnnvl_memory import MnnvlCheckpointCommunicator, MnnvlMemory from tensorrt_llm._torch.mnnvl_alltoall_workspace import _MnnvlAlltoAllWorkspaceLifecycle from tensorrt_llm.bindings import internal as _tllm_internal @@ -50,6 +40,7 @@ from ..moe_comm_timeout_guard import register_moe_comm_timeout_proxy from .base import Communication +from .execution_state import MoeExecutionState from .pull_combine import PULL_COMBINE_MIN_TOKENS, PullCombineSync, pull_reduce SYNC_PROTOCOL_ENV = "TRTLLM_NVLINK_ONE_SIDED_A2A_SYNC_PROTOCOL" @@ -248,7 +239,7 @@ def _sentinel_unsupported_reason( if use_low_precision_combine: return "it has no FP8 combine" if fault_tolerant: - return "the AlltoAll watchdog and rank-mask recovery observe fence completion flags only" + return "it takes no active-rank mask (Wide-EP fault tolerance)" if payload_format not in SENTINEL_PAYLOAD_FORMATS: return f"it does not carry the payload format {payload_format}" try: @@ -340,8 +331,13 @@ def _make_workspace_layout( dtype: torch.dtype, eplb_stats_num_experts: Optional[int], extra_payload_bytes_per_token: int, + rank_mask: bool = False, ) -> torch.Tensor: - """Plan per-rank control buffers and independently sized payload regions.""" + """Plan per-rank control buffers and independently sized payload regions. + + ``rank_mask`` reserves the incomplete-round word that the rank-mask kernels with an abort + state use. + """ if hidden_size <= 0 or extra_payload_bytes_per_token < 0: raise ValueError("hidden_size must be positive and extra payload size non-negative") tokens = ep_size * max_num_tokens @@ -368,6 +364,7 @@ def _make_workspace_layout( dispatch_bytes, combine_input_bytes, eplb_stats_num_experts, + rank_mask, ) @staticmethod @@ -376,6 +373,7 @@ def get_aux_data_size( max_num_tokens: int, eplb_stats_num_experts: Optional[int] = None, top_k: Optional[int] = None, + rank_mask: bool = False, ) -> int: """Control-buffer bytes; omitted top_k reserves the native routing limit.""" layout = torch.ops.trtllm.moe_a2a_get_workspace_layout( @@ -385,6 +383,7 @@ def get_aux_data_size( 0, 0, eplb_stats_num_experts, + rank_mask, ) return int(layout[_tllm_internal.thop.MOE_A2A_WORKSPACE_SIZE_INDEX]) @@ -429,10 +428,7 @@ def __init__( dtype: Optional[torch.dtype] = None, num_experts: Optional[int] = None, use_low_precision_combine: bool = False, - ep_group_health: EPGroupHealthLike | None = None, - alltoall_watchdog_timeout_s: Optional[float] = None, - alltoall_watchdog_poll_interval_s: float = DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - alltoall_watchdog_on_timeout: Optional[Callable[[AlltoAllWatchdogTimeout], None]] = None, + enable_wide_ep_fault_tolerance: bool = False, sentinel_payload_format: Optional[str] = "BF16", ) -> None: """ @@ -450,8 +446,8 @@ def __init__( ``sentinel_payload_format``; the fence transport takes the other steps. It needs SM100, SM103 or SM107, at most 64 EP ranks, BF16 expert outputs without FP8 combine, expert slots that split evenly across ranks, - and no AlltoAll watchdog or rank-mask recovery (which observe the fence - completion flags). TRTLLM_NVLINK_ONE_SIDED_A2A_SYNC_PROTOCOL=fence turns it + and no Wide-EP fault tolerance (its kernels take no active-rank mask). + TRTLLM_NVLINK_ONE_SIDED_A2A_SYNC_PROTOCOL=fence turns it off and =sentinel asks for it (a warning names the reason when it stays off); unset or "auto", it is on. TRTLLM_NVLINK_ONE_SIDED_A2A_SENTINEL_MAX_TOKENS sets the token limit, which otherwise is the largest step whose dispatch grid stays within one @@ -481,13 +477,11 @@ def __init__( use_low_precision_combine: If True, quantize the combine payload to FP8 for NVLink transfer (halves NVLink bandwidth usage, output precision is preserved). Corresponds to model_config.use_low_precision_moe_combine. - ep_group_health: Optional read-only committed EP membership. When present, rank-mask handling is - enabled in the CUDA kernels, and its mask defines the peers expected by the watchdog. Timeout - detection never mutates it. CUDA graphs are rejected until membership-scoped recapture lands. - alltoall_watchdog_timeout_s: Optional timeout for the host-side AlltoAll watchdog. If None, the - watchdog is disabled. - alltoall_watchdog_poll_interval_s: Poll interval for the watchdog thread. - alltoall_watchdog_on_timeout: Optional callback invoked when the watchdog reports suspects. + enable_wide_ep_fault_tolerance: Run the rank-mask kernels, which exchange only with the + EP ranks set in the device mask of the execution state that ``bind_execution_state`` + binds before the first dispatch, and stop waiting on peers once its abort is + requested. They read both when they run, so CUDA graphs follow their later changes. + Corresponds to model_config.enable_wide_ep_fault_tolerance. sentinel_payload_format: Dispatch payload format whose small steps the sentinel transport carries, one of SENTINEL_PAYLOAD_FORMATS (see sentinel_payload_format()), or None when the layer dispatches rows the transport does not carry (no sentinel regions then). Needs @@ -541,6 +535,7 @@ def __init__( dtype, self.eplb_stats_num_experts, 0, + rank_mask=enable_wide_ep_fault_tolerance, ) # Largest step of the sentinel transport; 0 when this instance does not use it. self.sentinel_max_tokens = 0 @@ -552,8 +547,7 @@ def __init__( hidden_size, dtype, sentinel_payload_format, - fault_tolerant=ep_group_health is not None - or alltoall_watchdog_timeout_s is not None, + fault_tolerant=enable_wide_ep_fault_tolerance, ) auto_workspace_size = int(metainfo[self.WORKSPACE_SIZE_INDEX]) workspace_mb_env = os.environ.get(_WORKSPACE_MB_ENV) @@ -586,6 +580,7 @@ def __init__( max_num_tokens_per_rank, self.eplb_stats_num_experts, self.top_k, + rank_mask=enable_wide_ep_fault_tolerance, ) # Split the rest between the dispatch payload and combine input regions. capacity = (self.workspace_size_per_rank - control_bytes) // 2 // alignment * alignment @@ -598,6 +593,7 @@ def __init__( capacity, capacity, self.eplb_stats_num_experts, + enable_wide_ep_fault_tolerance, ) # Fixed region capacities are shared by allocation, native bounds checks, # and workspace-backed views. Runtime shapes only pack tokens inside them. @@ -665,40 +661,18 @@ def __init__( self.mnnvl_mem = workspace_state["mnnvl_mem"] self.workspace = workspace_state["workspace"] self.max_num_tokens_per_rank = workspace_state["max_num_tokens_per_rank"] - self.ep_group_health = ep_group_health # Keep the kernel specialization stable for this communicator's lifetime. - self._rank_mask_enabled = ep_group_health is not None - if alltoall_watchdog_timeout_s is None and self.ep_group_health is not None: - alltoall_watchdog_timeout_s = DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S - flag_val_offset_index = self.FLAG_VAL_OFFSET_INDEX - dispatch_flags_offset_index = self.DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX - combine_flags_offset_index = self.COMBINE_COMPLETION_FLAGS_OFFSET_INDEX - if ( - flag_val_offset_index is None - or dispatch_flags_offset_index is None - or combine_flags_offset_index is None - ): - raise RuntimeError("MoE All-to-All metadata indices are not initialized") + self._rank_mask_enabled = enable_wide_ep_fault_tolerance + self.execution_state: Optional[MoeExecutionState] = None self._workspace_lifecycle = _MnnvlAlltoAllWorkspaceLifecycle.get_or_create( workspace_state=workspace_state, memory=self.mnnvl_mem, workspace=self.workspace, metainfo=workspace_state["metainfo"], - metainfo_index={ - "FLAG_VAL_OFFSET_INDEX": flag_val_offset_index, - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": dispatch_flags_offset_index, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": combine_flags_offset_index, - }, ep_rank=self.ep_rank, ep_size=self.ep_size, - health=self.ep_group_health, - ) - self._workspace_lifecycle.register( - self, - watchdog_timeout_s=alltoall_watchdog_timeout_s, - watchdog_poll_interval_s=alltoall_watchdog_poll_interval_s, - watchdog_on_timeout=alltoall_watchdog_on_timeout, ) + self._workspace_lifecycle.register(self) if workspace_created: NVLinkOneSided._WORKSPACES[self._workspace_key] = workspace_state NVLinkOneSided._WORKSPACE = workspace_state @@ -708,14 +682,13 @@ def __init__( self._workspace_registered = True # Two ranks may pull each other's BF16 partials straight from the workspace (pull_combine.py) - # once enable_pull_combine attaches the handshake. The watchdog watches only native kernels. + # once enable_pull_combine attaches the handshake. self._pull_sync: Optional[PullCombineSync] = None self._pull_combine_supported = ( self.ep_size == 2 and dtype == torch.bfloat16 and not self.use_low_precision_combine and not self._rank_mask_enabled - and alltoall_watchdog_timeout_s is None ) # Initialize dispatch state @@ -944,13 +917,55 @@ def row_bytes(tensor: Optional[torch.Tensor]) -> int: def moe_a2a_metainfo(self) -> torch.Tensor: return self._require_workspace_lifecycle().metainfo - @property - def _watchdog_coordinator(self) -> AlltoAllWatchdogCoordinator: - return self._require_workspace_lifecycle().coordinator + def bind_execution_state(self, state: MoeExecutionState) -> None: + """Run every later dispatch and combine with the device rank mask and abort state of ``state``. - @property - def _alltoall_watchdog(self) -> AlltoAllWatchdog | None: - return self._require_workspace_lifecycle().watchdog_for(self) + ``state.rearm`` then also resets this layer and this rank's row of its workspace. + """ + if not self._rank_mask_enabled: + raise ValueError( + "Bound execution requires an NVLinkOneSided built with " + "enable_wide_ep_fault_tolerance" + ) + if self.execution_state is not None: + raise RuntimeError("NVLinkOneSided is already bound to an execution state") + # The kernels treat the local rank as active, which the state checks for its own rank. + if state.ep_rank != self.ep_rank: + raise ValueError( + f"An execution state of EP rank {state.ep_rank} cannot run EP rank {self.ep_rank}" + ) + state.register_layer(self.reset_state) + state.register_workspace( + self.mnnvl_mem, self._reset_local_workspace, self._check_incomplete_round + ) + self.execution_state = state + + def _reset_local_workspace(self) -> None: + """Return this rank's row of the workspace to its first round, without a barrier.""" + torch.ops.trtllm.moe_a2a_initialize( + self.workspace, self.moe_a2a_metainfo, self.ep_rank, self.ep_size, barrier=False + ) + + def _check_incomplete_round(self) -> None: + """Enqueue the check of this rank's incomplete-round word in the workspace.""" + torch.ops.trtllm.moe_a2a_check_incomplete_round( + self.workspace, self.moe_a2a_metainfo, self.ep_rank, self.execution_state.abort + ) + + def _rank_mask_args(self) -> Dict[str, Optional[torch.Tensor]]: + """Return the rank-mask arguments of the all-to-all ops: the device mask that the rank-mask + kernels read and the abort state that ends their waits, or no mask without a rank mask.""" + if not self._rank_mask_enabled: + return {"active_rank_mask": None} + if self.execution_state is None: + raise RuntimeError( + "NVLinkOneSided with enable_wide_ep_fault_tolerance dispatches only once bound " + "(bind_execution_state)" + ) + return { + "active_rank_mask": self.execution_state.active_rank_mask, + "abort_state": self.execution_state.abort, + } def _require_workspace_lifecycle(self) -> _MnnvlAlltoAllWorkspaceLifecycle: lifecycle = self._workspace_lifecycle @@ -1112,11 +1127,10 @@ def dispatch( token_final_scales: Router weights [local_num_tokens, top_k] all_rank_num_tokens: Token counts per rank [ep_size] use_dp_padding: Whether to use DP padding (optional) - **kwargs: Strategy-specific arguments. In rank-mask mode, ``active_rank_mask`` may override the - committed membership for dispatch. Without an override, the committed mask and generation are - captured together; combine reuses that mask and fails closed if the generation changes first. - The masked kernel rejects inactive routes before remote access; that sentinel is an internal abort - artifact, not valid model output. + **kwargs: Strategy-specific arguments. With enable_wide_ep_fault_tolerance, the kernels + exchange only with the EP ranks set in the bound execution state's device mask and drop + the routes to the other ranks (their ``topk_target_ranks`` entries are -1). After an + abort, the outputs of the collective are invalid until ``rearm``. Returns: Tuple of (hidden_states, hidden_states_sf, token_selected_slots, token_final_scales) @@ -1129,7 +1143,6 @@ def dispatch( self._require_mapped() if self._dispatch_state.get("phase") == "dispatched": raise RuntimeError("dispatch called twice without an intervening combine") - reject_rank_mask_cuda_graph_capture(self._rank_mask_enabled) # Calculate runtime_max_tokens_per_rank from all_rank_num_tokens runtime_max_tokens_per_rank = max(all_rank_num_tokens) @@ -1163,13 +1176,6 @@ def dispatch( assert eplb_local_stats.size(0) == self.eplb_stats_num_experts, ( "eplb_local_stats size must match eplb_stats_num_experts" ) - requested_active_rank_mask = kwargs.get("active_rank_mask") - if not self._rank_mask_enabled and requested_active_rank_mask is not None: - raise ValueError("active_rank_mask requires committed EP group health") - active_rank_mask_snapshot = self._watchdog_coordinator.capture_active_rank_mask( - requested_active_rank_mask - ) - active_rank_mask = active_rank_mask_snapshot.active_rank_mask recv_buffers, combine_payload_offset, eplb_gathered_stats = ( torch.ops.trtllm.moe_a2a_dispatch( @@ -1187,22 +1193,16 @@ def dispatch( expert_id_payload_index=expert_id_payload_index if use_sentinel else None, invalid_token_expert_id=int(self.invalid_token_expert_id) if use_sentinel else None, enable_rank_mask=self._rank_mask_enabled, - active_rank_mask=active_rank_mask, + **self._rank_mask_args(), use_sentinel=use_sentinel, ) ) - # A sentinel round writes no completion flags for the watchdog to observe. - if not use_sentinel: - self._watchdog_coordinator.watch_collective( - self._alltoall_watchdog, "dispatch", active_rank_mask - ) if eplb_gathered_stats.numel() == 0: eplb_gathered_stats = None self._dispatch_state["eplb_gathered_stats"] = eplb_gathered_stats self._dispatch_state["combine_payload_offset"] = int(combine_payload_offset) self._dispatch_state["local_num_tokens"] = token_selected_slots.size(0) self._dispatch_state["runtime_max_tokens_per_rank"] = runtime_max_tokens_per_rank - self._dispatch_state["active_rank_mask_snapshot"] = active_rank_mask_snapshot self._dispatch_state["sentinel"] = use_sentinel self._dispatch_state["phase"] = "dispatched" @@ -1267,9 +1267,8 @@ def combine( final_hidden_states: Output from MoE computation Shape: [ep_size, max_tokens_per_rank, hidden_size] or [ep_size * max_tokens_per_rank, hidden_size] (will be reshaped) - **kwargs: Strategy-specific arguments. In rank-mask mode, ``active_rank_mask`` must match the mask - captured by dispatch for this collective when supplied. A committed-generation change since - dispatch aborts the collective epoch. + **kwargs: Strategy-specific arguments. With enable_wide_ep_fault_tolerance, the kernels + exchange only with the EP ranks set in the bound execution state's device mask. Returns: Combined output tensor [local_num_tokens, hidden_size] @@ -1278,7 +1277,6 @@ def combine( self._require_mapped() if self._dispatch_state.get("phase") != "dispatched": raise RuntimeError("combine called before a successful dispatch") - reject_rank_mask_cuda_graph_capture(self._rank_mask_enabled) local_num_tokens = self._dispatch_state.get("local_num_tokens") combine_payload_offset = self._dispatch_state.get("combine_payload_offset") @@ -1306,16 +1304,6 @@ def combine( raise ValueError( f"final_hidden_states must be 2D or 3D, got {final_hidden_states.dim()}D" ) - active_rank_mask_snapshot = self._dispatch_state.get("active_rank_mask_snapshot") - if not isinstance(active_rank_mask_snapshot, ActiveRankMaskSnapshot): - raise RuntimeError("combine called but dispatch rank-mask snapshot is missing") - requested_active_rank_mask = kwargs.get("active_rank_mask") - if not self._rank_mask_enabled and requested_active_rank_mask is not None: - raise ValueError("active_rank_mask requires committed EP group health") - active_rank_mask = self._watchdog_coordinator.active_rank_mask_for_combine( - active_rank_mask_snapshot, - requested_active_rank_mask, - ) # A sentinel dispatch is followed by a sentinel combine. use_sentinel = bool(self._dispatch_state.get("sentinel", False)) addend = kwargs.get("addend") @@ -1347,13 +1335,9 @@ def combine( bool(self.payload_in_workspace), use_low_precision=bool(self.use_low_precision_combine), enable_rank_mask=self._rank_mask_enabled, - active_rank_mask=active_rank_mask, + **self._rank_mask_args(), use_sentinel=use_sentinel, ) - if not use_sentinel: - self._watchdog_coordinator.watch_collective( - self._alltoall_watchdog, "combine", active_rank_mask - ) # Reset state for next round self.reset_state() diff --git a/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py b/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py index 5574ad91629c..c1d090fc51e8 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py +++ b/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py @@ -20,7 +20,6 @@ has committed as included vs. excluded from the data plane. It is consumed by: * AlltoAll communication backends (rank masking on dispatch / combine) - * The host-side AlltoAll watchdog (read-only expected-peer snapshot) * The MoE load balancer (emergency-mask reconfiguration) * The model engine and PyExecutor (degraded health reporting) diff --git a/tensorrt_llm/_torch/moe/fused_moe/wide_ep_ft.py b/tensorrt_llm/_torch/moe/fused_moe/wide_ep_ft.py deleted file mode 100644 index e0ee7e213a6d..000000000000 --- a/tensorrt_llm/_torch/moe/fused_moe/wide_ep_ft.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Shared WideEP fault-tolerance options for MoE communication paths.""" - -from __future__ import annotations - -import os -from typing import Any, Optional - -from tensorrt_llm._torch.alltoall_watchdog import ( - DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, -) - -from .ep_group_health import EPGroupHealth - -_ENABLE_ENV = "TLLM_FAULT_TOLERANCE_MODE" -_TIMEOUT_ENV = "TRTLLM_ALLTOALL_WATCHDOG_TIMEOUT_S" -_POLL_INTERVAL_ENV = "TRTLLM_ALLTOALL_WATCHDOG_POLL_INTERVAL_S" - -# This object contains membership committed by higher-layer recovery -# coordination. Detection threads must treat it as read-only. -_HEALTH_KEY = "wide_ep_ft_ep_group_health" -_TIMEOUT_KEY = "alltoall_watchdog_timeout_s" -_POLL_INTERVAL_KEY = "alltoall_watchdog_poll_interval_s" - - -def _env_enabled() -> bool: - return os.environ.get(_ENABLE_ENV) == "1" - - -def _float_option(extra_attrs: dict, key: str, env_name: str, default: float) -> float: - if key in extra_attrs: - return float(extra_attrs[key]) - if env_name in os.environ: - return float(os.environ[env_name]) - return default - - -def get_wide_ep_ft_options( - model_config: Any, -) -> tuple[Optional[EPGroupHealth], Optional[float], float]: - """Return committed EP membership and watchdog timing for a model. - - WideEP FT remains opt-in until the integration PR wires a public model - option. Callers can either inject ``wide_ep_ft_ep_group_health`` through - ``ModelConfig.extra_attrs`` or set ``TLLM_FAULT_TOLERANCE_MODE=1`` to create - one process-local membership object shared by all MoE communication layers. - The AlltoAll watchdog reads this object to determine expected peers and - reports suspects through its ``on_timeout`` seam; it never mutates the - committed membership directly. Rank-mask mode rejects CUDA graphs until - generation-scoped invalidation and recapture are implemented. - """ - - extra_attrs = getattr(model_config, "extra_attrs", {}) - health = extra_attrs.get(_HEALTH_KEY) or extra_attrs.get("ep_group_health") - rank_mask_enabled = health is not None or _env_enabled() - if rank_mask_enabled and getattr(model_config, "use_cuda_graph", False): - raise ValueError( - "WideEP fault tolerance does not support CUDA graphs until generation-scoped " - "invalidation and recapture are implemented" - ) - if health is None and rank_mask_enabled: - health = EPGroupHealth(model_config.mapping.moe_ep_size) - extra_attrs[_HEALTH_KEY] = health - - poll_interval_s = _float_option( - extra_attrs, - _POLL_INTERVAL_KEY, - _POLL_INTERVAL_ENV, - DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - ) - if health is None: - return None, None, poll_interval_s - - timeout_s = _float_option( - extra_attrs, - _TIMEOUT_KEY, - _TIMEOUT_ENV, - DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, - ) - return health, timeout_s, poll_interval_s diff --git a/tensorrt_llm/_torch/pyexecutor/error_classification.py b/tensorrt_llm/_torch/pyexecutor/error_classification.py index be10331f6a9e..257013a952d5 100644 --- a/tensorrt_llm/_torch/pyexecutor/error_classification.py +++ b/tensorrt_llm/_torch/pyexecutor/error_classification.py @@ -59,7 +59,6 @@ "nccl operation timed out", "alltoall timeout", "all-to-all timeout", - "alltoall watchdog", "completion_flags timeout", "deep_ep buffer barrier hang", "deepep buffer barrier hang", diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 8c4f46a21929..2acaef637ad6 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -1403,6 +1403,11 @@ def _timed_load( config) validate_lm_head_output_dtype(model, self.llm_args.lm_head_output_dtype) + if self.llm_args.enable_wide_ep_fault_tolerance: + # Imported here, so that a model without fault tolerance loads + # none of its modules. + from .rank_failure_model_load import prepare_model + prepare_model(model) # raise error when the built model cannot load the checkpoint as is _check_post_init_model_compat(model, load_format, checkpoint_loader) @@ -1738,6 +1743,8 @@ def _load_and_validate_config( checkpoint_loader), use_low_precision_moe_combine=self.llm_args.moe_config. use_low_precision_moe_combine, + enable_wide_ep_fault_tolerance=self.llm_args. + enable_wide_ep_fault_tolerance, nvfp4_gemm_allowed_backends=self.llm_args.nvfp4_gemm_config. allowed_backends, use_cute_dsl_blockscaling_mm=self.llm_args. diff --git a/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py new file mode 100644 index 000000000000..c61f04f67384 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Bind the MoE layers of a model to one execution state as it loads, under Wide-EP fault tolerance. + +The model loader calls ``prepare_model`` once the model is built, before any weight loads. +""" + +from __future__ import annotations + +from typing import List + +import torch + +from ..moe.fused_moe.communication.execution_state import MoeExecutionState +from ..moe.fused_moe.ep_group_health import EPGroupHealth +from ..moe.fused_moe.fused_moe_vanilla import VanillaMoE +from ..moe.fused_moe.interface import MoE + +# The alternative that each error about an unmet fault tolerance requirement offers. +_TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" + + +def prepare_model(model: torch.nn.Module) -> None: + """Bind every MoE layer of ``model`` to one execution state over all EP ranks. + + Raises: + ValueError: A MoE layer's communication cannot be bound, or the model has no MoE layer. + """ + comms = moe_comms(model) + mapping = model.model_config.mapping + state = MoeExecutionState( + EPGroupHealth(mapping.moe_ep_size), + mapping.moe_ep_rank, + torch.device("cuda", torch.cuda.current_device()), + ) + for comm in comms: + comm.bind_execution_state(state) + + +def moe_comms(model: torch.nn.Module) -> List[object]: + """Return the communication strategies that ``model``'s MoE layers use, each once. + + Raises: + ValueError: A MoE layer's communication cannot be bound to an execution state. + """ + comms = {} + for name, layer in model.named_modules(): + if not isinstance(layer, (MoE, VanillaMoE)): + continue + comm = getattr(layer, "comm", None) + if not hasattr(comm, "bind_execution_state"): + uses = "no communication strategy" if comm is None else type(comm).__name__ + raise ValueError( + f"MoE layer {name} ({type(layer).__name__}) uses {uses}, but Wide-EP fault " + "tolerance requires NVLinkOneSided, the only MoE communication that can leave " + f"out a failed rank. Select it or {_TURN_FAULT_TOLERANCE_OFF}." + ) + comms[id(comm)] = comm + if not comms: + raise ValueError( + f"Wide-EP fault tolerance requires a model with MoE layers; {_TURN_FAULT_TOLERANCE_OFF}" + ) + return list(comms.values()) diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index c4dc98f67864..8bbcb9c8cb17 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -8727,6 +8727,14 @@ def is_partial_model_loading(self) -> bool: "Only enable it if you intend to use this feature.", status="prototype") + enable_wide_ep_fault_tolerance: bool = Field( + default=False, + description="Run the MoE all-to-all of wide expert parallelism in its " + "fault-tolerant mode, in which the NVLink one-sided all-to-all " + "exchanges only with the EP ranks marked active. Requires the NVLink " + "one-sided all-to-all.", + status="prototype") + reorder_policy_config: Optional[ReorderRequestPolicyConfig] = Field( default=None, description="The request reordering policy to use.", diff --git a/tensorrt_llm/schemas/trtllm-serve-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-config.schema.json index bbbe5da6f77b..d40f08b33260 100644 --- a/tensorrt_llm/schemas/trtllm-serve-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-config.schema.json @@ -8940,6 +8940,13 @@ "title": "Enable Tokenization Cache", "type": "boolean" }, + "enable_wide_ep_fault_tolerance": { + "default": false, + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "status": "prototype", + "title": "Enable Wide Ep Fault Tolerance", + "type": "boolean" + }, "encode_only": { "default": false, "description": "Set to True to use the batch-forward encode() path, which runs a single forward pass and returns the model output directly, bypassing the scheduler and autoregressive loop. Works for encoder-only models (BERT, RoBERTa, reward models) and decoder models used in single-prefill mode (e.g., extracting embeddings). When False (default), uses the standard generate() path.", diff --git a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json index 05873393ab73..b802362accbb 100644 --- a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json @@ -2756,6 +2756,13 @@ "title": "Enable Tokenization Cache", "type": "boolean" }, + "enable_wide_ep_fault_tolerance": { + "default": false, + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "status": "prototype", + "title": "Enable Wide Ep Fault Tolerance", + "type": "boolean" + }, "encode_only": { "default": false, "description": "Set to True to use the batch-forward encode() path, which runs a single forward pass and returns the model output directly, bypassing the scheduler and autoregressive loop. Works for encoder-only models (BERT, RoBERTa, reward models) and decoder models used in single-prefill mode (e.g., extracting embeddings). When False (default), uses the standard generate() path.", @@ -11130,6 +11137,13 @@ "title": "Enable Tokenization Cache", "type": "boolean" }, + "enable_wide_ep_fault_tolerance": { + "default": false, + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "status": "prototype", + "title": "Enable Wide Ep Fault Tolerance", + "type": "boolean" + }, "encode_only": { "default": false, "description": "Set to True to use the batch-forward encode() path, which runs a single forward pass and returns the model output directly, bypassing the scheduler and autoregressive loop. Works for encoder-only models (BERT, RoBERTa, reward models) and decoder models used in single-prefill mode (e.g., extracting embeddings). When False (default), uses the standard generate() path.", diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index fcab39f80a57..e5a8f78fc483 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -490,6 +490,11 @@ "kind": "value", "path": "enable_tokenization_cache" }, + { + "capture_policy": "bool", + "kind": "value", + "path": "enable_wide_ep_fault_tolerance" + }, { "capture_policy": "bool", "kind": "value", diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index bbe81ace7898..08967afcef1a 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -280,6 +280,8 @@ l0_b200: - unittest/_torch/moe/test_moe_comm_timeout_guard.py - unittest/_torch/moe/fused_moe/test_deepgemm_fused_gather_finalize.py - unittest/_torch/moe/fused_moe/test_deepgemm_fused_expand_quant.py + - unittest/_torch/moe/test_moe_execution_state.py + - unittest/_torch/moe/test_moe_a2a_abort.py # ------------- MoE: test_moe_backend (by backend) --------------- - unittest/_torch/moe/test_moe_backend.py::test_cutlass_moe_routing_round_trip - unittest/_torch/moe/test_moe_backend.py::test_cutlass_moe_public_activation_ids diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index a4664b034c0c..d26fffa9269c 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -93,6 +93,10 @@ l0_gb200_multi_gpus: - unittest/_torch/multi_gpu/test_mnnvl_allreduce.py::test_mnnvl_checkpoint_rejects_wrong_group_membership - unittest/_torch/multi_gpu/test_mnnvl_allreduce.py::test_mnnvl_workspace_growth_keeps_captured_graphs - unittest/_torch/multi_gpu/test_mnnvl_allreduce.py::test_mnnvl_oneshot_early_pdl_release + - unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py::test_graph_replay_follows_installed_mask + - unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py::test_abort_releases_bound_collectives_waiting_on_a_stopped_rank + - unittest/_torch/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_rank_mask_all_active_matches_no_mask + - unittest/_torch/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_rank_mask_one_rank_masked_completes - unittest/_torch/modeling_v2/comm/test_modeling_v2_allgather_op_matrix.py - unittest/_torch/moe/test_moe_comm.py::TestMoEComm::test_mnnvl_checkpoint_preserves_moe_graph_addresses - unittest/_torch/moe/test_moe_comm.py::TestMoEComm::test_mnnvl_engine_checkpoint_coordination diff --git a/tests/unittest/_torch/modules/test_alltoall_watchdog.py b/tests/unittest/_torch/modules/test_alltoall_watchdog.py deleted file mode 100644 index a741d2527a52..000000000000 --- a/tests/unittest/_torch/modules/test_alltoall_watchdog.py +++ /dev/null @@ -1,669 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Unit tests for AlltoAllWatchdog (WideEP fault tolerance, PR 1a.4).""" - -import threading -import time -from collections.abc import Callable -from types import SimpleNamespace - -import pytest -import torch - -from tensorrt_llm._torch.alltoall_watchdog import ( - DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S, - DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S, - UNKNOWN_COMPLETION_FLAG, - AlltoAllWatchdog, - AlltoAllWatchdogCoordinator, - AlltoAllWatchdogTimeout, - CompletionFlagReadTimeout, - reject_rank_mask_cuda_graph_capture, -) -from tensorrt_llm._torch.moe.fused_moe.ep_group_health import EPGroupHealth -from tensorrt_llm._torch.moe.fused_moe.wide_ep_ft import get_wide_ep_ft_options - - -class FakeCompletionFlagReader: - """Thread-safe completion flag reader for pure-Python watchdog tests.""" - - def __init__(self, ep_size: int) -> None: - self._lock = threading.Lock() - self._flags = { - "dispatch": [0 for _ in range(ep_size)], - "combine": [0 for _ in range(ep_size)], - } - - def set_flags(self, phase: str, flags: list[int]) -> None: - with self._lock: - self._flags[phase] = list(flags) - - def read_completion_flags(self, phase: str) -> tuple[int, ...]: - with self._lock: - return tuple(self._flags[phase]) - - -class TimeoutCompletionFlagReader: - def read_completion_flags(self, phase: str) -> tuple[int, ...]: - raise CompletionFlagReadTimeout("blocked") - - -class OneGoodReadThenTimeoutReader: - def __init__(self, flags: tuple[int, ...]) -> None: - self._flags = flags - self._read_count = 0 - - def read_completion_flags(self, phase: str) -> tuple[int, ...]: - self._read_count += 1 - if self._read_count == 1: - return self._flags - raise CompletionFlagReadTimeout("blocked") - - -def _wait_for(predicate: Callable[[], bool], timeout_s: float = 1.0) -> None: - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.005) - raise AssertionError("condition was not reached before timeout") - - -def test_watchdog_completes_when_all_active_flags_arrive() -> None: - health = EPGroupHealth(4) - reader = FakeCompletionFlagReader(ep_size=4) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=4, - ep_rank=0, - completion_reader=reader, - timeout_s=0.2, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - reader.set_flags("dispatch", [1, 1, 1, 1]) - assert watchdog.wait_until_idle(timeout_s=1.0) - - assert events == [] - assert health.all_active() is True - - -def test_watchdog_completes_when_flags_advance_past_expected_generation() -> None: - health = EPGroupHealth(4) - reader = FakeCompletionFlagReader(ep_size=4) - reader.set_flags("dispatch", [2, 1, 5, 3]) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=4, - ep_rank=0, - completion_reader=reader, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - assert watchdog.wait_until_idle(timeout_s=1.0) - - assert events == [] - assert health.all_active() is True - - -def test_watchdog_handles_signed_uint32_generation_boundaries() -> None: - reader = FakeCompletionFlagReader(ep_size=2) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=2, - ep_rank=0, - completion_reader=reader, - timeout_s=0.05, - poll_interval_s=0.005, - on_timeout=events.append, - ) as watchdog: - reader.set_flags("dispatch", [-(1 << 31), -(1 << 31)]) - watchdog.watch(phase="dispatch", expected_flag=1 << 31) - assert watchdog.wait_until_idle(timeout_s=1.0) - - reader.set_flags("dispatch", [-1, -1]) - watchdog.watch(phase="dispatch", expected_flag=(1 << 32) - 1) - assert watchdog.wait_until_idle(timeout_s=1.0) - - reader.set_flags("dispatch", [0, 0]) - watchdog.watch(phase="dispatch", expected_flag=0) - assert watchdog.wait_until_idle(timeout_s=1.0) - - assert events == [] - - -def test_watchdog_defaults_match_design_doc() -> None: - reader = FakeCompletionFlagReader(ep_size=1) - watchdog = AlltoAllWatchdog(ep_size=1, ep_rank=0, completion_reader=reader) - - assert watchdog._timeout_s == DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S - assert watchdog._poll_interval_s == DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S - - -def test_watchdog_stop_is_terminal() -> None: - reader = FakeCompletionFlagReader(ep_size=1) - watchdog = AlltoAllWatchdog( - ep_size=1, - ep_rank=0, - completion_reader=reader, - timeout_s=0.2, - poll_interval_s=0.005, - ) - watchdog.start() - watchdog.stop(timeout_s=1.0) - - with pytest.raises(RuntimeError, match="stopped AlltoAllWatchdog"): - watchdog.start() - with pytest.raises(RuntimeError, match="stopped AlltoAllWatchdog"): - watchdog.watch(phase="dispatch", expected_flag=1) - - -def test_wide_ep_ft_options_create_shared_health_when_enabled( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("TLLM_FAULT_TOLERANCE_MODE", "1") - model_config = SimpleNamespace( - extra_attrs={}, - mapping=SimpleNamespace(moe_ep_size=4), - ) - - health, timeout_s, poll_interval_s = get_wide_ep_ft_options(model_config) - health_again, timeout_again_s, poll_again_s = get_wide_ep_ft_options(model_config) - - assert isinstance(health, EPGroupHealth) - assert health_again is health - assert timeout_s == DEFAULT_ALLTOALL_WATCHDOG_TIMEOUT_S - assert timeout_again_s == timeout_s - assert poll_interval_s == DEFAULT_ALLTOALL_WATCHDOG_POLL_INTERVAL_S - assert poll_again_s == poll_interval_s - - -def test_wide_ep_ft_options_reject_cuda_graphs( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("TLLM_FAULT_TOLERANCE_MODE", "1") - model_config = SimpleNamespace( - extra_attrs={}, - mapping=SimpleNamespace(moe_ep_size=4), - use_cuda_graph=True, - ) - - with pytest.raises(ValueError, match="does not support CUDA graphs"): - get_wide_ep_ft_options(model_config) - assert model_config.extra_attrs == {} - - -def test_rank_mask_mode_rejects_direct_cuda_graph_capture( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) - - reject_rank_mask_cuda_graph_capture(False) - with pytest.raises(RuntimeError, match="does not support CUDA graphs"): - reject_rank_mask_cuda_graph_capture(True) - - -def test_wide_ep_ft_options_ignore_legacy_enable_flag(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("TLLM_FAULT_TOLERANCE_MODE", raising=False) - monkeypatch.setenv("TRTLLM_ENABLE_WIDE_EP_FT", "1") - model_config = SimpleNamespace( - extra_attrs={}, - mapping=SimpleNamespace(moe_ep_size=4), - ) - - health, timeout_s, _ = get_wide_ep_ft_options(model_config) - - assert health is None - assert timeout_s is None - - -def test_watchdog_coordinator_reuses_committed_mask_when_generation_is_unchanged() -> None: - health = EPGroupHealth(4) - coordinator = AlltoAllWatchdogCoordinator( - workspace_state={}, - workspace=torch.zeros((4, 1), dtype=torch.uint8), - metainfo=torch.zeros((1,), dtype=torch.int64), - metainfo_index={}, - ep_rank=0, - health=health, - ) - dispatch_snapshot = coordinator.capture_active_rank_mask(None) - assert dispatch_snapshot.active_rank_mask is not None - assert dispatch_snapshot.committed_generation == 0 - - combine_mask = coordinator.active_rank_mask_for_combine(dispatch_snapshot, None) - - assert combine_mask is dispatch_snapshot.active_rank_mask - assert combine_mask.tolist() == [0b1111, 0, 0, 0] - - -def test_watchdog_coordinator_fails_closed_on_committed_generation_change() -> None: - health = EPGroupHealth(4) - coordinator = AlltoAllWatchdogCoordinator( - workspace_state={}, - workspace=torch.zeros((4, 1), dtype=torch.uint8), - metainfo=torch.zeros((1,), dtype=torch.int64), - metainfo_index={}, - ep_rank=0, - health=health, - ) - dispatch_snapshot = coordinator.capture_active_rank_mask(None) - - health.mark_failed(2) - health.mark_active(2) - assert health.get_mask() == 0b1111 - - with pytest.raises( - RuntimeError, - match="committed EP membership changed between dispatch and combine", - ): - coordinator.active_rank_mask_for_combine(dispatch_snapshot, None) - - -def test_watchdog_coordinator_converts_atomic_snapshot_to_mask_words() -> None: - health = EPGroupHealth(72) - health.mark_failed(70) - coordinator = AlltoAllWatchdogCoordinator( - workspace_state={}, - workspace=torch.zeros((72, 1), dtype=torch.uint8), - metainfo=torch.zeros((1,), dtype=torch.int64), - metainfo_index={}, - ep_rank=0, - health=health, - ) - - dispatch_snapshot = coordinator.capture_active_rank_mask(None) - - assert dispatch_snapshot.committed_generation == 1 - assert dispatch_snapshot.active_rank_mask is not None - assert dispatch_snapshot.active_rank_mask.tolist() == [(1 << 64) - 1, 0xBF, 0, 0] - - -def test_watchdog_coordinator_explicit_mask_is_not_bound_to_health_generation() -> None: - health = EPGroupHealth(4) - coordinator = AlltoAllWatchdogCoordinator( - workspace_state={}, - workspace=torch.zeros((4, 1), dtype=torch.uint8), - metainfo=torch.zeros((1,), dtype=torch.int64), - metainfo_index={}, - ep_rank=0, - health=health, - ) - explicit_mask = torch.tensor([0b1101, 0, 0, 0], dtype=torch.uint64) - dispatch_snapshot = coordinator.capture_active_rank_mask(explicit_mask) - assert dispatch_snapshot.committed_generation is None - - health.mark_failed(2) - combine_mask = coordinator.active_rank_mask_for_combine(dispatch_snapshot, None) - - assert combine_mask is dispatch_snapshot.active_rank_mask - assert combine_mask.tolist() == explicit_mask.tolist() - with pytest.raises(ValueError, match="mask captured at dispatch"): - coordinator.active_rank_mask_for_combine( - dispatch_snapshot, - torch.tensor(health.get_mask_words(), dtype=torch.uint64), - ) - - -def test_watchdog_no_detected_failure_publication_to_committed_health() -> None: - health = EPGroupHealth(4) - committed_before = health.snapshot() - reader = FakeCompletionFlagReader(ep_size=4) - reader.set_flags("dispatch", [1, 0, 1, 0]) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=4, - ep_rank=0, - completion_reader=reader, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - _wait_for(lambda: len(events) == 1) - assert watchdog.wait_until_idle(timeout_s=1.0) - - event = events[0] - assert event.phase == "dispatch" - assert event.expected_flag == 1 - assert event.observed_flags == (1, 0, 1, 0) - assert event.missing_ranks == (1, 3) - assert not hasattr(event, "marked_failed_ranks") - assert health.snapshot() == committed_before - - -def test_watchdog_ignores_ranks_already_failed_in_health_mask() -> None: - health = EPGroupHealth(4) - assert health.mark_failed(2) is True - reader = FakeCompletionFlagReader(ep_size=4) - reader.set_flags("dispatch", [1, 1, 0, 1]) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=4, - ep_rank=0, - completion_reader=reader, - timeout_s=0.05, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - assert watchdog.wait_until_idle(timeout_s=1.0) - - assert events == [] - assert health.get_failed_ranks() == frozenset({2}) - - -def test_watchdog_reports_local_missing_without_changing_committed_health() -> None: - health = EPGroupHealth(4) - reader = FakeCompletionFlagReader(ep_size=4) - reader.set_flags("combine", [0, 2, 2, 2]) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=4, - ep_rank=0, - completion_reader=reader, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="combine", expected_flag=2) - _wait_for(lambda: len(events) == 1) - - event = events[0] - assert event.missing_ranks == (0,) - assert health.get_failed_ranks() == frozenset() - - -def test_watchdog_poll_timeout_without_snapshot_fails_closed() -> None: - health = EPGroupHealth(3) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=3, - ep_rank=0, - completion_reader=TimeoutCompletionFlagReader(), - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - _wait_for(lambda: len(events) == 1) - - event = events[0] - assert event.poll_timed_out is True - assert event.observed_flags == (UNKNOWN_COMPLETION_FLAG,) * 3 - assert event.missing_ranks == (0, 1, 2) - assert health.all_active() is True - - -def test_watchdog_poll_timeout_with_prior_snapshot_does_not_mark_failed_rank() -> None: - health = EPGroupHealth(3) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=3, - ep_rank=0, - completion_reader=OneGoodReadThenTimeoutReader((1, 0, 1)), - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - _wait_for(lambda: len(events) == 1) - - event = events[0] - assert event.poll_timed_out is True - assert event.observed_flags == (1, 0, 1) - assert event.missing_ranks == (1,) - assert health.all_active() is True - - -def test_watchdog_callback_error_stops_and_clears_queue() -> None: - health = EPGroupHealth(2) - reader = FakeCompletionFlagReader(ep_size=2) - reader.set_flags("dispatch", [1, 0]) - - def raise_from_callback(event: AlltoAllWatchdogTimeout) -> None: - raise RuntimeError(f"callback failed for {event.phase}") - - with AlltoAllWatchdog( - ep_size=2, - ep_rank=0, - completion_reader=reader, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=raise_from_callback, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - _wait_for(lambda: watchdog.last_error is not None) - assert watchdog.wait_until_idle(timeout_s=1.0) - assert isinstance(watchdog.last_error, RuntimeError) - with pytest.raises(RuntimeError, match="stopped AlltoAllWatchdog"): - watchdog.watch(phase="dispatch", expected_flag=2) - - -def test_watchdog_preserves_fifo_order_and_clears_followups_after_timeout() -> None: - health = EPGroupHealth(3) - reader = FakeCompletionFlagReader(ep_size=3) - reader.set_flags("dispatch", [1, 0, 1]) - reader.set_flags("combine", [0, 0, 0]) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog( - ep_size=3, - ep_rank=0, - completion_reader=reader, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=1) - watchdog.watch(phase="combine", expected_flag=2) - _wait_for(lambda: len(events) == 1) - assert watchdog.wait_until_idle(timeout_s=1.0) - time.sleep(0.05) - - assert len(events) == 1 - assert events[0].phase == "dispatch" - assert events[0].missing_ranks == (1,) - assert health.get_failed_ranks() == frozenset() - - -def test_watchdog_from_workspace_reads_phase_specific_offsets() -> None: - ep_size = 3 - ep_rank = 1 - workspace = torch.zeros((ep_size, 64), dtype=torch.uint8) - metainfo = torch.zeros((10,), dtype=torch.int64) - metainfo_index = { - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": 4, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": 5, - } - metainfo[4] = 4 - metainfo[5] = 20 - workspace[ep_rank, 4:16].view(torch.int32).copy_(torch.tensor([7, 7, 7], dtype=torch.int32)) - workspace[ep_rank, 20:32].view(torch.int32).copy_(torch.tensor([0, 8, 8], dtype=torch.int32)) - health = EPGroupHealth(ep_size) - events: list[AlltoAllWatchdogTimeout] = [] - - with AlltoAllWatchdog.from_workspace( - workspace=workspace, - metainfo=metainfo, - metainfo_index=metainfo_index, - ep_rank=ep_rank, - ep_size=ep_size, - timeout_s=0.02, - poll_interval_s=0.005, - health=health, - on_timeout=events.append, - ) as watchdog: - watchdog.watch(phase="dispatch", expected_flag=7) - assert watchdog.wait_until_idle(timeout_s=1.0) - - watchdog.watch(phase="combine", expected_flag=8) - _wait_for(lambda: len(events) == 1) - - assert events[0].phase == "combine" - assert events[0].missing_ranks == (0,) - assert health.get_failed_ranks() == frozenset() - - -def test_workspace_coordinators_share_fifo_watchdog() -> None: - ep_size = 3 - ep_rank = 0 - workspace_state: dict[str, object] = {} - workspace = torch.zeros((ep_size, 64), dtype=torch.uint8) - metainfo = torch.tensor([0, 4, 16], dtype=torch.int64) - metainfo_index = { - "FLAG_VAL_OFFSET_INDEX": 0, - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": 1, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": 2, - } - workspace[ep_rank, 4:16].view(torch.int32).copy_(torch.tensor([1, 0, 1])) - health = EPGroupHealth(ep_size) - events: list[AlltoAllWatchdogTimeout] = [] - on_timeout = events.append - coordinators = [ - AlltoAllWatchdogCoordinator( - workspace_state=workspace_state, - workspace=workspace, - metainfo=metainfo, - metainfo_index=metainfo_index, - ep_rank=ep_rank, - health=health, - ) - for _ in range(2) - ] - watchdogs = [ - coordinator.acquire_watchdog( - ep_size=ep_size, - timeout_s=0.02, - poll_interval_s=0.005, - on_timeout=on_timeout, - ) - for coordinator in coordinators - ] - - try: - assert watchdogs[0] is watchdogs[1] - coordinators[0].watch_collective(watchdogs[0], "dispatch", None) - coordinators[1].watch_collective(watchdogs[1], "combine", None) - _wait_for(lambda: len(events) == 1) - assert watchdogs[0].wait_until_idle(timeout_s=1.0) - time.sleep(0.05) - - assert len(events) == 1 - assert events[0].phase == "dispatch" - assert events[0].missing_ranks == (1,) - assert health.get_failed_ranks() == frozenset() - finally: - for coordinator, watchdog in zip(coordinators, watchdogs): - coordinator.release_watchdog(watchdog) - - -def test_workspace_coordinator_wraps_shared_generation() -> None: - workspace_state: dict[str, object] = {} - workspace = torch.zeros((1, 32), dtype=torch.uint8) - workspace[0, 0:4].view(torch.int32).fill_(-1) - metainfo = torch.tensor([0, 4, 8], dtype=torch.int64) - metainfo_index = { - "FLAG_VAL_OFFSET_INDEX": 0, - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": 1, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": 2, - } - coordinator = AlltoAllWatchdogCoordinator( - workspace_state=workspace_state, - workspace=workspace, - metainfo=metainfo, - metainfo_index=metainfo_index, - ep_rank=0, - ) - watchdog = coordinator.acquire_watchdog( - ep_size=1, - timeout_s=0.05, - poll_interval_s=0.005, - ) - - try: - coordinator.watch_collective(watchdog, "dispatch", None) - assert watchdog.wait_until_idle(timeout_s=1.0) - finally: - coordinator.release_watchdog(watchdog) - - -def test_unmonitored_coordinator_advances_shared_generation() -> None: - workspace_state: dict[str, object] = {} - workspace = torch.zeros((2, 32), dtype=torch.uint8) - metainfo = torch.tensor([0, 4, 12], dtype=torch.int64) - metainfo_index = { - "FLAG_VAL_OFFSET_INDEX": 0, - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": 1, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": 2, - } - coordinators = [ - AlltoAllWatchdogCoordinator( - workspace_state=workspace_state, - workspace=workspace, - metainfo=metainfo, - metainfo_index=metainfo_index, - ep_rank=0, - ) - for _ in range(2) - ] - events: list[AlltoAllWatchdogTimeout] = [] - watchdog = coordinators[0].acquire_watchdog( - ep_size=2, - timeout_s=0.02, - poll_interval_s=0.005, - on_timeout=events.append, - ) - - try: - coordinators[1].watch_collective(None, "dispatch", None) - coordinators[0].watch_collective(watchdog, "dispatch", None) - _wait_for(lambda: len(events) == 1) - assert events[0].expected_flag == 2 - finally: - coordinators[0].release_watchdog(watchdog) - - -def test_watchdog_rejects_active_mask_without_local_rank() -> None: - reader = FakeCompletionFlagReader(ep_size=4) - with AlltoAllWatchdog( - ep_size=4, - ep_rank=2, - completion_reader=reader, - timeout_s=0.1, - poll_interval_s=0.005, - ) as watchdog: - with pytest.raises(ValueError, match="local ep_rank"): - watchdog.watch(phase="dispatch", expected_flag=1, active_mask=0b1011) diff --git a/tests/unittest/_torch/moe/moe_a2a_test_utils.py b/tests/unittest/_torch/moe/moe_a2a_test_utils.py new file mode 100644 index 000000000000..950f0bfc8c71 --- /dev/null +++ b/tests/unittest/_torch/moe/moe_a2a_test_utils.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Helpers of the tests of the MoE All-to-All ops, with every logical peer on one GPU.""" + +import torch + +from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided +from tensorrt_llm.bindings import internal + +CAPACITY, HIDDEN = 8, 128 + + +def a2a_workspace(ep_size: int, tokens: int = CAPACITY) -> tuple[torch.Tensor, ...]: + meta = NVLinkOneSided._make_workspace_layout( + ep_size, 1, tokens, HIDDEN, torch.bfloat16, None, 32, rank_mask=True + ) + size = int(meta[int(internal.thop.MOE_A2A_WORKSPACE_SIZE_INDEX)]) + workspace = torch.zeros((ep_size, size), dtype=torch.uint8, device="cuda") + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, 0, ep_size, barrier=False) + experts = torch.zeros((tokens, 1), dtype=torch.int32, device="cuda") + payload = torch.arange(tokens * HIDDEN, dtype=torch.bfloat16, device="cuda") + return workspace, meta, experts, payload.reshape(tokens, HIDDEN) + + +def a2a_dispatch(workspace, meta, experts, controls, payload, tokens=CAPACITY, rank=0): + """Dispatch [payload, experts] like the MoE layer as ``rank``.""" + ep_size = workspace.size(0) + args = (experts, [payload, experts], workspace, meta, tokens, rank, ep_size, 1, ep_size, None) + return torch.ops.trtllm.moe_a2a_dispatch(*args, **controls) + + +def a2a_combine(workspace, meta, payload, offset, controls, tokens=CAPACITY, rank=0): + """Combine like the MoE layer as ``rank``.""" + args = (payload, tokens, workspace, meta, tokens, rank, workspace.size(0), 1, offset, False) + return torch.ops.trtllm.moe_a2a_combine(*args, **controls) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py b/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py index 1869c211ba93..5b504277fddf 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py @@ -794,107 +794,260 @@ def test_nvlink_one_sided(case: Case, mpi_pools: dict[tuple[int, bool], MPIPoolE # ============================================================================ -# Dispatch completion flags for the host watchdog +# CUDA graphs under an execution state # ============================================================================ -def _run_watchdog_worker(_unused: int) -> dict: - """Dispatch once with a watchdog armed, and report what it saw.""" +def _run_graph_replay_worker(failed_rank: int) -> dict: + """Capture one round trip with every rank active, then replay it without ``failed_rank``.""" # Import locally so cloudpickle does not serialize torch's dynamic ops namespace. - import os - import torch - # The default policy: a watchdog communicator must keep the sentinel transport off by itself. - os.environ.pop(SYNC_PROTOCOL_ENV, None) - - from tensorrt_llm._torch.moe.fused_moe.wide_ep_ft import EPGroupHealth + from tensorrt_llm._torch.moe.fused_moe.communication.execution_state import MoeExecutionState + from tensorrt_llm._torch.moe.fused_moe.ep_group_health import EPGroupHealth rank = MPI.COMM_WORLD.Get_rank() ep_size = MPI.COMM_WORLD.Get_size() torch.cuda.set_device(rank) MnnvlMemory.initialize() + if not all(MPI.COMM_WORLD.allgather(MnnvlMemory.supports_mnnvl())): + return {"skip": "NVLink one-sided is not supported on every participating GPU"} + + hidden, capacity, slots_per_rank = 256, 8, 2 + health = EPGroupHealth(ep_size) + state = MoeExecutionState(health, rank, torch.device("cuda", rank)) + mapping = Mapping(rank=rank, world_size=ep_size, tp_size=ep_size, moe_ep_size=ep_size) + comm = NVLinkOneSided( + mapping=mapping, + num_slots=ep_size * slots_per_rank, + top_k=1, + max_num_tokens_per_rank=capacity, + hidden_size=hidden, + dtype=torch.bfloat16, + enable_wide_ep_fault_tolerance=True, + ) + comm.bind_execution_state(state) + everyone = list(range(ep_size)) + active = [r for r in everyone if r != failed_rank] + x = torch.empty(capacity, hidden, dtype=torch.bfloat16, device="cuda") + slots = torch.empty(capacity, 1, dtype=torch.int32, device="cuda") + weights = torch.ones(capacity, 1, device="cuda") + errors = [] + + def check(ok: bool, what: str) -> None: + if not ok: + errors.append(f"rank {rank}: {what}") + + def set_inputs(owners: list[int], seed: int) -> None: + """Route this rank's tokens, in turn, to the first slot of each of ``owners``.""" + generator = torch.Generator().manual_seed(seed * ep_size + rank) + x.copy_(torch.randn(capacity, hidden, generator=generator).bfloat16()) + owner = torch.tensor(owners)[torch.arange(capacity) % len(owners)] + slots.copy_((owner * slots_per_rank)[:, None].int()) + + def round_trip() -> torch.Tensor: + """The experts return their input, so a complete round combines each token to itself.""" + recv = comm.dispatch(x, None, slots, weights, [capacity] * ep_size) + return comm.combine(recv[0]) + + try: + set_inputs(everyone, 0) + check(torch.equal(round_trip(), x), "eager round combined wrong") + torch.cuda.synchronize() + MPI.COMM_WORLD.Barrier() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = round_trip() + set_inputs(everyone, 1) + graph.replay() + torch.cuda.synchronize() + check(torch.equal(output, x), "replay with every rank combined wrong") + MPI.COMM_WORLD.Barrier() + + # The failed rank replays nothing more; the others install the mask without it and + # replay the graph captured before. + health.mark_failed(failed_rank) + if rank in active: + address = state.active_rank_mask.data_ptr() + state.install_mask() + check(state.active_rank_mask.data_ptr() == address, "the mask moved") + for seed in (2, 3): + set_inputs(active, seed) + graph.replay() + torch.cuda.synchronize() + check(torch.equal(output, x), f"replay {seed} without rank {failed_rank} wrong") + MPI.COMM_WORLD.Barrier() + return {"rank": rank, "errors": errors, "sentinel_max_tokens": comm.sentinel_max_tokens} + finally: + comm.destroy() + + +def test_graph_replay_follows_installed_mask() -> None: + """A CUDA graph captured with every rank active replays without a rank masked out later.""" + ep_size, failed_rank = 4, 2 + if torch.cuda.device_count() < ep_size: + pytest.skip(f"requires {ep_size} GPUs") + # A replay that waited for the masked rank would end at this kernel timeout. + executor = MPIPoolExecutor(ep_size, env={"TRTLLM_MOE_COMM_TIMEOUT_SEC": "60"}) + try: + results = list(executor.map(_run_graph_replay_worker, [failed_rank] * ep_size)) + skipped = [r["skip"] for r in results if "skip" in r] + if skipped: + pytest.skip(skipped[0]) + assert [e for r in results for e in r["errors"]] == [] + # The sentinel transport takes no active-rank mask, so every step takes the fence one. + assert all(r["sentinel_max_tokens"] == 0 for r in results) + finally: + executor.shutdown(wait=True, cancel_futures=True) + +# ============================================================================ +# Abort of bound collectives stuck on a rank that stopped +# ============================================================================ + + +def _run_abort_stuck_collective_worker(failed_rank: int, stop_before: str, round_flag: int) -> dict: + """Abort a round from ``round_flag`` whose ``failed_rank`` stops before ``stop_before``.""" + # Import locally so cloudpickle does not serialize torch's dynamic ops namespace. + import time + + import torch + + from tensorrt_llm._torch.moe.fused_moe.communication.execution_state import MoeExecutionState + from tensorrt_llm._torch.moe.fused_moe.ep_group_health import EPGroupHealth + from tensorrt_llm.bindings import internal + + rank = MPI.COMM_WORLD.Get_rank() + ep_size = MPI.COMM_WORLD.Get_size() + torch.cuda.set_device(rank) + MnnvlMemory.initialize() if not all(MPI.COMM_WORLD.allgather(MnnvlMemory.supports_mnnvl())): return {"skip": "NVLink one-sided is not supported on every participating GPU"} - timeouts: list = [] + hidden, capacity, slots_per_rank = 256, 8, 2 + evidence = int(internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX) + health = EPGroupHealth(ep_size) mapping = Mapping(rank=rank, world_size=ep_size, tp_size=ep_size, moe_ep_size=ep_size) comm = NVLinkOneSided( mapping=mapping, - num_slots=ep_size * 2, + num_slots=ep_size * slots_per_rank, top_k=1, - max_num_tokens_per_rank=8, - hidden_size=512, + max_num_tokens_per_rank=capacity, + hidden_size=hidden, dtype=torch.bfloat16, - ep_group_health=EPGroupHealth(ep_size), - alltoall_watchdog_timeout_s=5.0, - alltoall_watchdog_poll_interval_s=0.02, - alltoall_watchdog_on_timeout=timeouts.append, + enable_wide_ep_fault_tolerance=True, ) + everyone = list(range(ep_size)) + active = [r for r in everyone if r != failed_rank] + errors = [] + + def check(ok: bool, what: str) -> None: + if not ok: + errors.append(f"rank {rank}: {what}") + + def dispatch(senders: list[int], seed: int) -> tuple[torch.Tensor, tuple]: + """Dispatch this rank's tokens, each to the first slot of one of ``senders``.""" + generator = torch.Generator().manual_seed(seed * ep_size + rank) + x = torch.randn(capacity, hidden, generator=generator).bfloat16().cuda() + owner = torch.tensor(senders)[torch.arange(capacity) % len(senders)] + slots = (owner * slots_per_rank)[:, None].int().cuda() + weights = torch.ones(capacity, 1, device="cuda") + counts = [capacity if r in senders else 0 for r in everyone] + return x, comm.dispatch(x, None, slots, weights, counts) + + def round_trip(senders: list[int], seed: int) -> tuple[torch.Tensor, torch.Tensor]: + """The experts return their input, so a complete round combines each token to itself.""" + x, recv = dispatch(senders, seed) + return x, comm.combine(recv[0], all_rank_max_num_tokens=capacity) + try: - num_tokens = 4 - slots = torch.randint(0, ep_size * 2, (num_tokens, 1), dtype=torch.int32, device="cuda") - payload = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device="cuda") - scales = torch.ones(num_tokens, 1, dtype=torch.float32, device="cuda") - comm.dispatch(payload, None, slots, scales, [num_tokens] * ep_size) + state = MoeExecutionState(health, rank, torch.device("cuda")) + comm.bind_execution_state(state) + MPI.COMM_WORLD.Barrier() + x, combined = round_trip(everyone, 0) + check(torch.equal(combined, x), "healthy round combined wrong") torch.cuda.synchronize() + MPI.COMM_WORLD.Barrier() - # The flags the watchdog polls live in this rank's own workspace row. - offset = comm.moe_a2a_metainfo[comm.DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX].item() - flags = ( - comm.workspace[comm.ep_rank, offset : offset + ep_size * 4] - .view(torch.int32) - .cpu() - .tolist() - ) + # Round 1: the dispatch waits for flag value round_flag + 1 and the combine for + # round_flag + 2, modulo 2**32. A wait publishes 1 for a flag value that wrapped to 0. flag_val_offset = comm.moe_a2a_metainfo[comm.FLAG_VAL_OFFSET_INDEX].item() - expected = int( - comm.workspace[comm.ep_rank, flag_val_offset : flag_val_offset + 4] - .view(torch.int32) - .cpu() - .item() + comm.workspace[comm.ep_rank, flag_val_offset : flag_val_offset + 4].view(torch.int32).fill_( + round_flag - 2**32 if round_flag >= 2**31 else round_flag ) - MPI.COMM_WORLD.barrier() - return { - "rank": rank, - "flags": flags, - "expected_flag": expected, - "timeouts": [t.missing_ranks for t in timeouts], - "sentinel_max_tokens": comm.sentinel_max_tokens, - } + if rank == failed_rank: + if stop_before == "combine": + dispatch(everyone, 1) + torch.cuda.synchronize() + comm.reset_state() + else: + x, combined = round_trip(everyone, 1) + done = torch.cuda.Event() + done.record() + stuck_on = (round_flag + (1 if stop_before == "dispatch" else 2)) % 2**32 or 1 + deadline = time.monotonic() + 10 + while ( + state.waited_flags().get(failed_rank) != stuck_on + and not done.query() + and time.monotonic() < deadline + ): + time.sleep(0.01) + check( + state.waited_flags().get(failed_rank) == stuck_on, + f"no wait evidence on rank {failed_rank}: " + f"{state.abort[evidence : evidence + ep_size].tolist()}", + ) + check(not done.query(), "the round completed without the stopped rank") + check(not state.aborted(), "aborted before the request") + state.request_abort() + start = time.monotonic() + torch.cuda.synchronize() + check(time.monotonic() - start < 10, "the abort did not release the waiting collective") + check(state.abort[:2].tolist() == [1, 1], f"abort state {state.abort[:2].tolist()}") + check(not state.abort[evidence:].any(), "wait evidence left after the abort") + # The combine of an incomplete round takes only the tokens this rank routed to itself. + expected = torch.zeros_like(x) + own = torch.arange(capacity, device=x.device) % ep_size == rank + expected[own] = x[own] + check(torch.equal(combined, expected), "the incomplete round combined wrong") + MPI.COMM_WORLD.Barrier() + + health.mark_failed(failed_rank) + if rank in active: + state.rearm() + MPI.COMM_WORLD.Barrier() + if rank in active: + state.install_mask() + for seed in (2, 3): + x, combined = round_trip(active, seed) + check(torch.equal(combined, x), f"round {seed} after the recovery combined wrong") + torch.cuda.synchronize() + check(not state.aborted(), "a round after the recovery aborted") + MPI.COMM_WORLD.Barrier() + return {"rank": rank, "errors": errors} finally: comm.destroy() -@pytest.mark.parametrize("ep_size", [2, 4]) -def test_dispatch_publishes_watchdog_flags(ep_size: int) -> None: - """Dispatch must publish completion flags so the watchdog sees live peers. +@pytest.mark.parametrize("round_flag", [2, 2**32 - 2], ids=["no_wrap", "flag_wraps"]) +@pytest.mark.parametrize("stop_before", ["dispatch", "combine"]) +def test_abort_releases_bound_collectives_waiting_on_a_stopped_rank( + stop_before: str, round_flag: int +) -> None: + """Abort a bound dispatch or combine waiting on a stopped rank, then recover without it. - The host watchdog observes only the completion flag table. Without the - flags a healthy collective is reported as every peer missing. + With ``flag_wraps`` the combine waits for round flag value 0, which is still published. """ + ep_size, failed_rank = 4, 2 if torch.cuda.device_count() < ep_size: pytest.skip(f"requires {ep_size} GPUs") - executor = MPIPoolExecutor(ep_size) + executor = MPIPoolExecutor(ep_size, env={"TRTLLM_MOE_COMM_TIMEOUT_SEC": "30"}) try: - results = list(executor.map(_run_watchdog_worker, range(ep_size))) + args = [(failed_rank, stop_before, round_flag)] * ep_size + results = list(executor.starmap(_run_abort_stuck_collective_worker, args)) skipped = [r["skip"] for r in results if "skip" in r] if skipped: pytest.skip(skipped[0]) - results.sort(key=lambda r: r["rank"]) - for result in results: - rank = result["rank"] - expected = result["expected_flag"] - assert result["flags"] == [expected] * ep_size, ( - f"rank {rank}: dispatch completion flags {result['flags']} did not all " - f"reach {expected}; the watchdog would report healthy peers as missing" - ) - assert result["timeouts"] == [], ( - f"rank {rank}: watchdog reported {result['timeouts']} on a healthy collective" - ) - # Sentinel steps write no completion flags, so the watchdog keeps them off. - assert result["sentinel_max_tokens"] == 0, f"rank {rank}: sentinel on with a watchdog" - + assert [e for r in results for e in r["errors"]] == [] finally: executor.shutdown(wait=True, cancel_futures=True) diff --git a/tests/unittest/_torch/moe/test_communication_factory.py b/tests/unittest/_torch/moe/test_communication_factory.py index 6d8f5af02ee2..4a18f85adcba 100644 --- a/tests/unittest/_torch/moe/test_communication_factory.py +++ b/tests/unittest/_torch/moe/test_communication_factory.py @@ -59,6 +59,7 @@ def _make_model_config( use_cuda_graph=False, use_low_precision_moe_combine=False, moe_load_balancer=None, + enable_wide_ep_fault_tolerance=False, ) @@ -574,6 +575,62 @@ def fail_if_called(*args, **kwargs): assert isinstance(strategy, AllGatherReduceScatter) +def _create_one_sided(model_config, forced: bool): + if forced: + return communication_factory.CommunicationFactory._create_forced_method( + "NVLINK_ONE_SIDED", + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + payload_in_workspace=False, + alltoall_result_do_sum=True, + hidden_size=4096, + ) + return communication_factory.CommunicationFactory.create_strategy( + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + hidden_size=4096, + ) + + +@pytest.mark.parametrize("fault_tolerance", [False, True]) +@pytest.mark.parametrize("forced", [False, True]) +def test_one_sided_runs_rank_mask_kernels_only_with_fault_tolerance( + monkeypatch: pytest.MonkeyPatch, forced: bool, fault_tolerance: bool +): + """NVLinkOneSided runs the rank-mask kernels exactly when the model is fault tolerant.""" + model_config = _make_model_config() + model_config.enable_wide_ep_fault_tolerance = fault_tolerance + one_sided = Mock() + monkeypatch.setattr(communication_factory, "NVLinkOneSided", one_sided) + + assert _create_one_sided(model_config, forced) is one_sided.return_value + assert one_sided.call_args.kwargs["enable_wide_ep_fault_tolerance"] is fault_tolerance + + +@pytest.mark.parametrize("fault_tolerance", [False, True]) +def test_fault_tolerance_reports_one_sided_error( + monkeypatch: pytest.MonkeyPatch, fault_tolerance: bool +): + """Fault tolerance runs only on NVLinkOneSided, so its construction error is not swallowed.""" + model_config = _make_model_config(torch.float16) + model_config.enable_wide_ep_fault_tolerance = fault_tolerance + monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) + monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) + monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") + + if fault_tolerance: + with pytest.raises(ValueError, match="requires the NVLinkOneSided.*strategy unavailable"): + _create_one_sided(model_config, forced=False) + else: + assert isinstance(_create_one_sided(model_config, forced=False), AllGatherReduceScatter) + + _NCCL_EP_QUANT_ALGOS = [ None, QuantAlgo.W4A16_MXFP4, diff --git a/tests/unittest/_torch/moe/test_moe_a2a_abort.py b/tests/unittest/_torch/moe/test_moe_a2a_abort.py new file mode 100644 index 000000000000..9e5cd14bce20 --- /dev/null +++ b/tests/unittest/_torch/moe/test_moe_a2a_abort.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run-time rank mask and abort of the fence MoE All-to-All ops. + +Every logical peer lives in one allocation on one GPU, and the tests issue the ops of each rank in +turn, so they check masking, polling and the order of the ranks' steps only. +""" + +import ctypes +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +import torch +from _torch.moe.moe_a2a_test_utils import CAPACITY, HIDDEN, a2a_combine, a2a_dispatch, a2a_workspace + +import tensorrt_llm # noqa: F401 +from tensorrt_llm.bindings import internal + +# The abort state holds ``requested`` and ``status``, then the flag value awaited from each peer. +_ABORT_WORDS = int(internal.thop.MOE_A2A_ABORT_STATE_WORDS) + + +def _offset(meta: torch.Tensor, name: str) -> int: + return int(meta[int(getattr(internal.thop, f"MOE_A2A_{name}_OFFSET_INDEX"))]) + + +def _word(workspace: torch.Tensor, offset: int, row: int = 0) -> torch.Tensor: + return workspace[row, offset : offset + 4].view(torch.int32) + + +def _mask_words(mask: int) -> torch.Tensor: + """Return the rank mask as the kernels' four little-endian uint64 words.""" + return torch.tensor([(mask >> (64 * w)) & (2**64 - 1) for w in range(4)], dtype=torch.uint64) + + +def _rank_mask_controls(mask: int) -> tuple[dict, torch.Tensor]: + """Return the rank-mask controls of a dispatch or combine and their pinned abort state.""" + abort = torch.zeros(_ABORT_WORDS, dtype=torch.int32).pin_memory() + words = _mask_words(mask).cuda() + return {"enable_rank_mask": True, "active_rank_mask": words, "abort_state": abort}, abort + + +def _next_parity(workspace: torch.Tensor, meta: torch.Tensor, rank: int = 0) -> tuple[int, int]: + """Return the flag value and receive-count parity of ``rank``'s next round.""" + expected_flag = _word(workspace, _offset(meta, "FLAG_VAL"), row=rank).item() + 1 + return expected_flag, ((expected_flag - 1) >> 1) & 1 + + +def _finish(abort: torch.Tensor, failure: str, timeout_s: float = 10) -> None: + """Wait for the queued work. At the deadline, release it through the abort state and fail.""" + done = torch.cuda.Event() + done.record() + deadline = time.monotonic() + timeout_s + while not done.query(): + if time.monotonic() > deadline: + abort[0] = 1 + torch.cuda.synchronize() + pytest.fail(failure) + time.sleep(0.01) + + +def test_cancel_and_exclusion_on_replay() -> None: + """One captured graph observes the abort state and the mask written before each replay.""" + workspace, meta, experts, payload = a2a_workspace(2) + controls, abort = _rank_mask_controls(3) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + received, offset, _ = a2a_dispatch(workspace, meta, experts, controls, payload) + output = a2a_combine(workspace, meta, received[0], offset, controls) + + # Peer 1 never signals. A pending abort request releases the dispatch's wait, and the combine of + # the incomplete round takes only this rank's own rows. + abort[0] = 1 + graph.replay() + _finish(abort, "the abort request did not release the dispatch's wait") + assert abort[:2].tolist() == [1, 1] + assert _word(workspace, _offset(meta, "INCOMPLETE_ROUND")).item() == 1 + torch.testing.assert_close(output, payload, rtol=0, atol=0) + + # Until the workspace is reset, rank 0 publishes round flags only to itself, so peer 1 never + # combines outputs of the incomplete round. Withdrawing the abort request does not change that. + def peer_flags() -> list[int]: + """Return rank 0's dispatch and combine flags in peer 1's row.""" + regions = ("DISPATCH_COMPLETION_FLAGS", "COMBINE_COMPLETION_FLAGS") + return [_word(workspace, _offset(meta, region), row=1).item() for region in regions] + + published = peer_flags() + assert published == [1, 0] + # Peer 1 takes part in the next round with no rows for rank 0. + expected_flag, parity = _next_parity(workspace, meta) + _word(workspace, _offset(meta, "RECV_COUNTERS") + 4 * (parity * 2 + 1)).fill_(0) + _word(workspace, _offset(meta, "DISPATCH_COMPLETION_FLAGS") + 4).fill_(expected_flag) + _word(workspace, _offset(meta, "COMBINE_COMPLETION_FLAGS") + 4).fill_(expected_flag + 1) + abort.zero_() + graph.replay() + _finish(abort, "the round after the incomplete one did not complete with peer 1's flags") + assert peer_flags() == published + + # Rearmed with peer 1 excluded, the same graph completes without it. + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, 0, 2, barrier=False) + abort.zero_() + controls["active_rank_mask"].copy_(_mask_words(1)) + for _ in range(2): + graph.replay() + _finish(abort, "the replay with peer 1 excluded waited for it") + torch.testing.assert_close(output, payload, rtol=0, atol=0) + assert torch.all(received[1][0] == 0) + assert abort[:2].tolist() == [0, 0] + + +@pytest.mark.parametrize("ep_size, rank", [(2, 0), (72, 70)], ids=["first_word", "second_word"]) +def test_no_peer_writes_after_incomplete_round(ep_size: int, rank: int) -> None: + """Until the workspace reset, a dispatch writes nothing into rows a peer may still read.""" + workspace, meta, experts, payload = a2a_workspace(ep_size) + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, rank, ep_size, barrier=False) + controls, abort = _rank_mask_controls((1 << ep_size) - 1) + abort[0] = 1 # No peer signals, so the round stays incomplete. + a2a_dispatch(workspace, meta, experts, controls, payload, rank=rank) + _finish(abort, "the abort request did not release the dispatch's wait") + assert _word(workspace, _offset(meta, "INCOMPLETE_ROUND"), row=rank).item() == 1 + + abort.zero_() + peers = torch.cat([workspace[:rank], workspace[rank + 1 :]]) + # Route the tokens to peers of both mask words. + experts.copy_( + torch.tensor([0, 65, 71, 64, 3, 70, 69, 1], dtype=torch.int32).view(-1, 1) % ep_size + ) + a2a_dispatch(workspace, meta, experts, controls, payload, rank=rank) + _finish(abort, "the dispatch after the incomplete round waited for a peer") + assert torch.equal(torch.cat([workspace[:rank], workspace[rank + 1 :]]), peers) + + +@pytest.mark.parametrize( + "ep_size, rank, peer", [(2, 0, 1), (72, 34, 2)], ids=["two_ranks", "rank_34_of_72"] +) +def test_aborted_dispatch_uses_no_rows_of_unconfirmed_peers( + ep_size: int, rank: int, peer: int +) -> None: + """An aborted round uses no rows of a peer whose completion flag it did not see. + + The peer writes its receive count but never its flag, and again after the abort. Rank 34 of 72 + waits on peer 2 and then on itself in the same lane, and keeps its own rows. + """ + workspace, meta, experts, payload = a2a_workspace(ep_size) + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, rank, ep_size, barrier=False) + experts.fill_(rank) + controls, abort = _rank_mask_controls((1 << ep_size) - 1) + _, parity = _next_parity(workspace, meta, rank) + counts = _offset(meta, "RECV_COUNTERS") + 4 * parity * ep_size + count = _word(workspace, counts + 4 * peer, row=rank) + count.fill_(CAPACITY) + abort[0] = 1 + received, offset, _ = a2a_dispatch(workspace, meta, experts, controls, payload, rank=rank) + _finish(abort, "the abort request did not release the dispatch's wait") + assert count.item() == 0 + assert _word(workspace, counts + 4 * rank, row=rank).item() == CAPACITY + + count.fill_(CAPACITY) + expert_ids = received[1] + expert_ids[peer].fill_(12345) # an expert id that no rank owns + torch.ops.trtllm.moe_a2a_sanitize_expert_ids(expert_ids, workspace, meta, rank, -1) + assert torch.all(expert_ids[peer] == -1) + assert torch.equal(expert_ids[rank], experts) + + # The rows of the peer in this rank's combine input region, which the peer reads. + stride = HIDDEN * payload.element_size() + first = offset + peer * CAPACITY * stride + peer_rows = workspace[rank, first : first + CAPACITY * stride] + peer_rows.fill_(0x7F) + output = a2a_combine(workspace, meta, received[0], offset, controls, rank=rank) + _finish(abort, "the combine of the incomplete round waited for the peer") + assert torch.all(peer_rows == 0x7F) + torch.testing.assert_close(output, payload, rtol=0, atol=0) + + +def _check_incomplete_round(workspace, meta, abort, rank=0) -> None: + """Run the incomplete-round check of ``rank`` that follows a forward.""" + torch.ops.trtllm.moe_a2a_check_incomplete_round(workspace, meta, rank, abort) + _finish(abort, f"rank {rank}'s incomplete-round check did not finish") + + +@pytest.mark.parametrize("peer_aborts", [False, True], ids=["healthy", "peer_aborts"]) +def test_incomplete_round_check_cancels_on_overwritten_peer_rows(peer_aborts: bool) -> None: + """The check after a combine that read rows a peer overwrote after an aborted dispatch records a cancel. + + Rank 0's tokens go to rank 1, whose MoE output is its combine input region, as with zero-copy + combine. Rank 1 completes its round before rank 0 combines it. With ``peer_aborts``, rank 1 + then aborts its next dispatch and overwrites that region like its next MoE. The combine + records nothing; the check that follows the forward records the cancel. + """ + workspace, meta, experts, payload = a2a_workspace(2) + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, 1, 2, barrier=False) + controls, abort = _rank_mask_controls(3) + peer_controls, peer_abort = _rank_mask_controls(3) + experts.fill_(1) + # Rank 1's dispatch flag for rank 0, and rank 0's combine flag for rank 1. + dispatch_flag, _ = _next_parity(workspace, meta) + _word(workspace, _offset(meta, "DISPATCH_COMPLETION_FLAGS") + 4).fill_(dispatch_flag) + _word(workspace, _offset(meta, "COMBINE_COMPLETION_FLAGS"), row=1).fill_(dispatch_flag + 1) + received, offset, _ = a2a_dispatch(workspace, meta, experts, controls, payload) + peer_received, _, _ = a2a_dispatch(workspace, meta, experts, peer_controls, payload, rank=1) + moe_output = torch.ops.trtllm.moe_a2a_get_combine_payload_tensor( + workspace, meta, 1, 2, CAPACITY, payload.dtype, HIDDEN + ).view(2, CAPACITY, HIDDEN) + moe_output.copy_(peer_received[0]) + a2a_combine(workspace, meta, moe_output, offset, peer_controls, rank=1) + if peer_aborts: + peer_abort[0] = 1 + a2a_dispatch(workspace, meta, experts, peer_controls, payload, rank=1) + moe_output.fill_(-1) + _finish(peer_abort, "rank 1's round waited for rank 0") + output = a2a_combine(workspace, meta, received[0], offset, controls) + _finish(abort, "rank 0's combine waited for rank 1") + expected = torch.full_like(payload, -1) if peer_aborts else payload + torch.testing.assert_close(output, expected, rtol=0, atol=0) + assert abort[:2].tolist() == [0, 0] + _check_incomplete_round(workspace, meta, abort) + assert abort[:2].tolist() == [0, int(peer_aborts)] + + +def test_own_round_value_keeps_peer_rows() -> None: + """A word holding the round's own combine value keeps peer rows valid in the sanitize and the prepare. + + A peer whose combine of this round aborted stores that value with its next dispatch. The combine then + exchanges with every peer, so the rows it publishes must hold the peers' tokens, and the check + after it records a cancel. + """ + workspace, meta, experts, payload = a2a_workspace(2) + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, 1, 2, barrier=False) + controls, _ = _rank_mask_controls(3) + peer_controls, peer_abort = _rank_mask_controls(3) + experts.fill_(1) + # Rank 1's dispatch flag for rank 0, and rank 0's combine flag for rank 1. + dispatch_flag, _ = _next_parity(workspace, meta, rank=1) + _word(workspace, _offset(meta, "DISPATCH_COMPLETION_FLAGS") + 4).fill_(dispatch_flag) + _word(workspace, _offset(meta, "COMBINE_COMPLETION_FLAGS"), row=1).fill_(dispatch_flag + 1) + a2a_dispatch(workspace, meta, experts, controls, payload) + peer_received, offset, _ = a2a_dispatch( + workspace, meta, experts, peer_controls, payload, rank=1 + ) + # Rank 0's combine of this round aborted, and its next dispatch marked rank 1. + _word(workspace, _offset(meta, "INCOMPLETE_ROUND"), row=1).fill_(dispatch_flag + 2) + expert_ids = peer_received[1].clone() + torch.ops.trtllm.moe_a2a_sanitize_expert_ids(expert_ids, workspace, meta, 1, -1) + torch.testing.assert_close(expert_ids, peer_received[1], rtol=0, atol=0) + a2a_combine(workspace, meta, peer_received[0].clone(), offset, peer_controls, rank=1) + _finish(peer_abort, "rank 1's combine waited for rank 0") + rows = torch.ops.trtllm.moe_a2a_get_combine_payload_tensor( + workspace, meta, 1, 2, CAPACITY, payload.dtype, HIDDEN + ).view(2, CAPACITY, HIDDEN) + torch.testing.assert_close(rows[0], peer_received[0][0], rtol=0, atol=0) + # The combine published its readiness to rank 0, so it exchanged with every peer. + ready = _word(workspace, _offset(meta, "COMBINE_COMPLETION_FLAGS") + 4) + assert ready.item() == dispatch_flag + 1 + assert peer_abort[:2].tolist() == [0, 0] + _check_incomplete_round(workspace, meta, peer_abort, rank=1) + assert peer_abort[:2].tolist() == [0, 1] + + +def test_abort_state_must_be_pinned() -> None: + """The kernels read the abort state at its host address, so the ops refuse pageable memory.""" + workspace, meta, experts, payload = a2a_workspace(2) + controls, _ = _rank_mask_controls(3) + controls["abort_state"] = torch.zeros(_ABORT_WORDS, dtype=torch.int32) + with pytest.raises(RuntimeError, match="pinned"): + a2a_dispatch(workspace, meta, experts, controls, payload) + + +def test_incomplete_dispatch_marks_active_peers() -> None: + """A dispatch whose round is incomplete marks the rounds of the active peers incomplete.""" + workspace, meta, experts, payload = a2a_workspace(3) + controls, abort = _rank_mask_controls(0b011) # rank 2 is inactive + incomplete_round = _offset(meta, "INCOMPLETE_ROUND") + # The incomplete-round value of a peer's aborted dispatch with flag value 5. + _word(workspace, incomplete_round).fill_(5) + a2a_dispatch(workspace, meta, experts, controls, payload) + _finish(abort, "the dispatch of an incomplete round waited for a peer") + assert [_word(workspace, incomplete_round, row=r).item() for r in range(3)] == [5, 5, 0] + + +def _fence_timeout() -> None: + """Check that a fence wait on a silent peer still fails at the timeout.""" + torch.ops.trtllm.moe_a2a_set_timeout(1) + workspace, meta, experts, payload = a2a_workspace(2) + controls, _ = _rank_mask_controls(3) + a2a_dispatch(workspace, meta, experts, controls, payload) + try: + torch.cuda.synchronize() + except RuntimeError: + # Exit at once, flushing the device printf C stdio holds on a pipe. + print("fence timeout trapped", flush=True) + ctypes.CDLL(None).fflush(None) + os._exit(0) + raise AssertionError("dispatch completed without its peer") + + +def test_fence_timeout_still_traps() -> None: + # The child traps on purpose, in a fresh interpreter without the launcher's environment. + launcher_prefixes = ("SLURM_", "PMIX_", "PMI_", "OMPI_") + env = {k: v for k, v in os.environ.items() if not k.startswith(launcher_prefixes)} + # The child imports the shared helpers of the unit tests, as pytest makes them importable. + unittest_root = str(Path(__file__).resolve().parents[2]) + env["PYTHONPATH"] = os.pathsep.join(p for p in (unittest_root, env.get("PYTHONPATH")) if p) + result = subprocess.run( + [sys.executable, __file__], env=env, capture_output=True, text=True, timeout=300 + ) + assert result.returncode == 0, result.stdout + result.stderr + # The kernel's own report, not just any device error. + assert "dispatch: rank 0 timed out waiting for rank 1 flag=" in result.stdout, result.stdout + assert "fence timeout trapped" in result.stdout, result.stdout + + +if __name__ == "__main__": + _fence_timeout() diff --git a/tests/unittest/_torch/moe/test_moe_a2a_fake_ops.py b/tests/unittest/_torch/moe/test_moe_a2a_fake_ops.py new file mode 100644 index 000000000000..169e9febf7fa --- /dev/null +++ b/tests/unittest/_torch/moe/test_moe_a2a_fake_ops.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Check that the fake implementations of the MoE all-to-all ops accept every schema argument.""" + +import pytest +import torch +from torch._subclasses import FakeTensorMode + +import tensorrt_llm # noqa: F401 + +pytestmark = pytest.mark.cpu_only + + +def test_moe_a2a_fake_ops_accept_every_schema_argument(): + """Pass the rank-mask arguments as keywords, as a bound NVLinkOneSided layer does. + + They are not their defaults, so the dispatcher hands all of them to the fake implementations. + """ + ep_rank, ep_size, max_tokens, top_k, num_experts, hidden = 0, 2, 4, 2, 8, 16 + with FakeTensorMode(): + experts = torch.empty(max_tokens, top_k, dtype=torch.int32) + payload = torch.empty(max_tokens, hidden) + workspace = torch.empty(ep_size, 1024, dtype=torch.uint8) + metainfo = torch.empty(16, dtype=torch.int64) + rank_mask = { + "enable_rank_mask": True, + "active_rank_mask": torch.empty(1, dtype=torch.uint64), + "abort_state": torch.empty(16, dtype=torch.int32), + } + torch.ops.trtllm.moe_a2a_initialize(workspace, metainfo, ep_rank, ep_size, barrier=False) + layout = (workspace, metainfo, max_tokens, ep_rank, ep_size, top_k) + # EPLB stats, expert-id payload index and invalid token expert id. + dispatch_options = (None, 0, num_experts) + recv, offset, _ = torch.ops.trtllm.moe_a2a_dispatch( + experts, [payload], *layout, num_experts, *dispatch_options, **rank_mask + ) + # payload_in_workspace and use_low_precision. + combine_options = (False, False) + output = torch.ops.trtllm.moe_a2a_combine( + recv[0], max_tokens, *layout, offset, *combine_options, **rank_mask + ) + torch.ops.trtllm.moe_a2a_check_incomplete_round( + workspace, metainfo, ep_rank, rank_mask["abort_state"] + ) + assert recv[0].shape == (ep_size, max_tokens, hidden) + assert output.shape == (max_tokens, hidden) diff --git a/tests/unittest/_torch/moe/test_moe_comm.py b/tests/unittest/_torch/moe/test_moe_comm.py index 1bbd10b13057..b3f4a8b5ee8f 100644 --- a/tests/unittest/_torch/moe/test_moe_comm.py +++ b/tests/unittest/_torch/moe/test_moe_comm.py @@ -229,11 +229,11 @@ def _safe_cpu(t: Optional[torch.Tensor]) -> Optional[torch.Tensor]: def _ep_mask_words(ep_size: int, dead_ranks: Set[int]) -> torch.Tensor: - """Build the CPU active-rank mask tensor expected by moe_a2a ops.""" + """Build the device active-rank mask tensor that moe_a2a ops read when their kernels run.""" health = EPGroupHealth(ep_size) for rank in dead_ranks: health.mark_failed(rank) - return torch.tensor(health.get_mask_words(), dtype=torch.uint64, device="cpu") + return torch.tensor(health.get_mask_words(), dtype=torch.uint64, device="cuda") def _make_rank_mask_payload(local_num_tokens: int, hidden_size: int, rank: int) -> torch.Tensor: @@ -287,6 +287,9 @@ def _run_nvlink_rank_mask_dispatch( active_rank_mask: Optional[torch.Tensor], ) -> Tuple[List[torch.Tensor], int, torch.Tensor, torch.Tensor]: """Run raw NVLink one-sided dispatch with an optional active rank mask.""" + # Import locally so cloudpickle does not serialize torch's dynamic ops namespace. + import torch + recv_tensors, combine_payload_offset, _ = torch.ops.trtllm.moe_a2a_dispatch( token_selected_experts, [payload], @@ -325,6 +328,9 @@ def _run_nvlink_rank_mask_combine( active_rank_mask: Optional[torch.Tensor], ) -> torch.Tensor: """Run raw NVLink one-sided combine with an optional active rank mask.""" + # Import locally so cloudpickle does not serialize torch's dynamic ops namespace. + import torch + return torch.ops.trtllm.moe_a2a_combine( combine_payload, local_num_tokens, @@ -1364,6 +1370,16 @@ def _worker_rank_mask_all_active_matches_no_mask(config: CommTestConfig) -> dict enable_rank_mask=True, active_rank_mask=None, ) + # The kernels read the mask when they run, so it must be in device memory. + with pytest.raises(RuntimeError, match="must be on the workspace device"): + _run_nvlink_rank_mask_dispatch( + comm, + token_selected_experts, + payload, + local_num_tokens, + enable_rank_mask=True, + active_rank_mask=all_active_mask.cpu(), + ) out_no_mask, topk_no_mask, _ = _run_nvlink_rank_mask_dispatch_combine( comm, @@ -1448,7 +1464,9 @@ def _worker_rank_mask_one_rank_masked( dtype=torch.int32, device="cuda", ) - _, _, masked_target_ranks, masked_send_indices = _run_nvlink_rank_mask_dispatch( + # Every dispatch needs its combine: the next round's flag values and + # receive-counter bank assume the two alternate. + _, masked_target_ranks, masked_send_indices = _run_nvlink_rank_mask_dispatch_combine( comm, masked_routes, _make_rank_mask_payload(local_num_tokens, config.hidden_size, rank), diff --git a/tests/unittest/_torch/moe/test_moe_execution_state.py b/tests/unittest/_torch/moe/test_moe_execution_state.py new file mode 100644 index 000000000000..2d5f01000c89 --- /dev/null +++ b/tests/unittest/_torch/moe/test_moe_execution_state.py @@ -0,0 +1,370 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MoeExecutionState, and the NVLinkOneSided layers bound to it at model load.""" + +import re +import threading +import time +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from _torch.moe.moe_a2a_test_utils import a2a_combine, a2a_dispatch, a2a_workspace + +import tensorrt_llm # noqa: F401 +from tensorrt_llm._torch.moe.fused_moe import ConfigurableMoE, TritonFusedMoE, VanillaMoE +from tensorrt_llm._torch.moe.fused_moe.communication import AllGatherReduceScatter +from tensorrt_llm._torch.moe.fused_moe.communication.execution_state import MoeExecutionState +from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided +from tensorrt_llm._torch.moe.fused_moe.ep_group_health import EPGroupHealth +from tensorrt_llm._torch.pyexecutor import rank_failure_model_load +from tensorrt_llm._torch.pyexecutor.rank_failure_model_load import moe_comms, prepare_model +from tensorrt_llm.bindings import internal +from tensorrt_llm.mapping import Mapping + +_EP_SIZE, _TOKENS, _HIDDEN = 2, 8, 128 +# Word of the abort state holding the round flag awaited from peer 0. +_WAITED_FLAGS = int(internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX) +_JOIN_TIMEOUT_S = 10.0 + + +def _bindable_state(**attributes) -> SimpleNamespace: + """Return a stand-in for an execution state that records the layers bound to it.""" + return SimpleNamespace(register_layer=Mock(), register_workspace=Mock(), **attributes) + + +def _collective_round(state: MoeExecutionState, workspace, meta, experts, payload) -> torch.Tensor: + """Run dispatch and combine as rank 0 under ``state``, and return the combined output.""" + controls = { + "enable_rank_mask": True, + "active_rank_mask": state.active_rank_mask, + "abort_state": state.abort, + } + received, offset, _ = a2a_dispatch(workspace, meta, experts, controls, payload) + return a2a_combine(workspace, meta, received[0], offset, controls) + + +def _wait_for(condition, state: MoeExecutionState, failure: str, timeout_s: float = 10) -> None: + """Wait until ``condition()`` holds. At the deadline, abort the collectives and fail.""" + deadline = time.monotonic() + timeout_s + while not condition(): + if time.monotonic() > deadline: + state.request_abort() + torch.cuda.synchronize() + pytest.fail(failure) + time.sleep(0.01) + + +def _queued_work_done(): + done = torch.cuda.Event() + done.record() + return done.query + + +def test_mask_holds_the_active_ranks() -> None: + """Rank 70 of 72 is bit 6 of the second mask word.""" + health = EPGroupHealth(72) + health.mark_failed(70) + state = MoeExecutionState(health, 0, torch.device("cuda")) + assert state.active_rank_mask.is_cuda + assert state.active_rank_mask.cpu().tolist() == [(1 << 64) - 1, 0xBF, 0, 0] + + +def test_install_mask_rewrites_the_mask_in_place() -> None: + """The device mask changes only with the generation of the EP group health, at one address.""" + health = EPGroupHealth(4) + state = MoeExecutionState(health, 0, torch.device("cuda")) + address = state.active_rank_mask.data_ptr() + + def install() -> list[int]: + state.install_mask() + return state.active_rank_mask.cpu().tolist() + + assert install() == [0b1111, 0, 0, 0] + health.mark_failed(3) + assert install() == [0b0111, 0, 0, 0] + # The kernels treat this rank as active, so a mask without it is refused. + health.mark_failed(0) + with pytest.raises(RuntimeError, match="must mark the local ep_rank"): + install() + assert state.active_rank_mask.data_ptr() == address + assert state.active_rank_mask.cpu().tolist() == [0b0111, 0, 0, 0] + + +def test_state_requires_active_local_rank() -> None: + excluded = EPGroupHealth(2) + excluded.mark_failed(0) + with pytest.raises(RuntimeError, match="must mark the local ep_rank"): + MoeExecutionState(excluded, 0, torch.device("cuda")) + + +def test_abort_and_rearm_after_peer_failure() -> None: + """A dispatch stuck on a failed peer aborts from another thread, and runs again after rearm.""" + workspace, meta, experts, payload = a2a_workspace(_EP_SIZE) + health = EPGroupHealth(_EP_SIZE) + state = MoeExecutionState(health, 0, workspace.device) + resets = [] + + def reset_workspace() -> None: + resets.append(1) + torch.ops.trtllm.moe_a2a_initialize(workspace, meta, 0, _EP_SIZE, barrier=False) + + state.register_workspace(object(), reset_workspace, Mock()) + + _collective_round(state, workspace, meta, experts, payload) + done = _queued_work_done() + # A long wait names the flag value it expects from rank 1. It is not an abort. + _wait_for(state.waited_flags, state, "the dispatch published no waited flag") + assert state.waited_flags() == {1: 1} + assert not state.aborted() + requester = threading.Thread(target=state.request_abort) + requester.start() + requester.join(_JOIN_TIMEOUT_S) + assert not requester.is_alive(), "abort request queued behind the spinning kernel" + _wait_for(done, state, "collective kept waiting after the abort request") + assert state.aborted() + assert state.waited_flags() == {} + + health.mark_failed(1) + with pytest.raises(RuntimeError, match="abort state"): + state.install_mask() + assert state.active_rank_mask.cpu().tolist() == [3, 0, 0, 0] + state.rearm() + assert resets == [1] + assert not state.aborted() + state.install_mask() + assert state.active_rank_mask.cpu().tolist() == [1, 0, 0, 0] + for _ in range(2): + output = _collective_round(state, workspace, meta, experts, payload) + _wait_for(_queued_work_done(), state, "the round without rank 1 waited for it") + torch.testing.assert_close(output, payload, rtol=0, atol=0) + assert not state.aborted() + + +def test_waited_flags_map_peer_to_flag() -> None: + """waited_flags() maps each peer a kernel waits for to the round flag value it waits for.""" + state = MoeExecutionState(EPGroupHealth(4), 0, torch.device("cuda")) + assert state.waited_flags() == {} + state.abort[_WAITED_FLAGS + 1] = 7 # the dispatch of round 3 + state.abort[_WAITED_FLAGS + 3] = 8 # the combine of round 3 + state.abort[_WAITED_FLAGS + 2] = -(1 << 31) # uint32 flag 1 << 31, past the int32 range + assert state.waited_flags() == {1: 7, 2: 1 << 31, 3: 8} + + +def test_incomplete_round_word_marks_the_state_aborted() -> None: + """check_incomplete_rounds checks each bound workspace once, and a set word cancels the state.""" + state = MoeExecutionState(EPGroupHealth(_EP_SIZE), 0, torch.device("cuda")) + checks = [] + bound = [] + for name in ("first", "second"): + workspace, meta, _, _ = a2a_workspace(_EP_SIZE) + incomplete_round = int(meta[int(internal.thop.MOE_A2A_INCOMPLETE_ROUND_OFFSET_INDEX)]) + bound.append(workspace[0, incomplete_round : incomplete_round + 4].view(torch.int32)) + + def check(name=name, workspace=workspace, meta=meta) -> None: + checks.append(name) + torch.ops.trtllm.moe_a2a_check_incomplete_round(workspace, meta, 0, state.abort) + + key = object() + # Layers that share a workspace register it again. + for _ in range(2): + state.register_workspace(key, Mock(), check) + + state.check_incomplete_rounds() + torch.cuda.synchronize() + assert checks == ["first", "second"] + assert not state.aborted() + + bound[1].fill_(1) + state.check_incomplete_rounds() + torch.cuda.synchronize() + assert state.aborted() + assert state.abort[:2].tolist() == [0, 1] + + +def test_rearm_resets_bound_layers_and_shared_workspaces() -> None: + """Rearm resets each shared workspace once and every layer an abort left between dispatch and combine.""" + state = MoeExecutionState(EPGroupHealth(2), 0, torch.device("cuda")) + resets = [] + shared = object() + for name in ("shared", "shared again"): + state.register_workspace(shared, lambda name=name: resets.append(name), Mock()) + comm = _frontend() + comm._reset_local_workspace = lambda: resets.append("layer workspace") + comm.bind_execution_state(state) + comm._dispatch_state = {"phase": "dispatched"} + state.rearm() + assert resets == ["shared", "layer workspace"] + assert comm._dispatch_state == {"phase": "idle"} + + +def test_graph_replay_follows_installed_mask() -> None: + """A CUDA graph captured with every rank active replays without the rank masked out later.""" + workspace, meta, experts, payload = a2a_workspace(_EP_SIZE) + health = EPGroupHealth(_EP_SIZE) + state = MoeExecutionState(health, 0, workspace.device) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = _collective_round(state, workspace, meta, experts, payload) + + health.mark_failed(1) + state.install_mask() + for _ in range(2): + graph.replay() + _wait_for(_queued_work_done(), state, "the replay waited for the masked-out rank") + torch.testing.assert_close(output, payload, rtol=0, atol=0) + assert not state.aborted() + + +def _frontend(*, fault_tolerance: bool = True, ep_rank: int = 0) -> NVLinkOneSided: + """Build an NVLinkOneSided with only the state that dispatch and combine read.""" + comm = NVLinkOneSided.__new__(NVLinkOneSided) + comm._rank_mask_enabled = fault_tolerance + comm.execution_state = None + comm._dispatch_state = {"phase": "idle"} + comm._workspace_lifecycle = Mock() + comm.mnnvl_mem = Mock(mapped=True) + comm._sentinel_row = None + comm.enable_eplb = False + comm.workspace = torch.zeros(1, dtype=torch.uint8) + comm.ep_rank, comm.ep_size, comm.top_k, comm.num_experts = ep_rank, _EP_SIZE, 1, _EP_SIZE + comm.invalid_token_expert_id = -1 + comm.payload_in_workspace = False + comm.use_low_precision_combine = False + return comm + + +@pytest.mark.cpu_only +def test_bind_refuses_unsupported_layers() -> None: + state = _bindable_state(ep_rank=0) + comm = _frontend() + comm.bind_execution_state(state) + with pytest.raises(RuntimeError, match="already bound"): + comm.bind_execution_state(state) + with pytest.raises(ValueError, match="built with enable_wide_ep_fault_tolerance"): + _frontend(fault_tolerance=False).bind_execution_state(state) + with pytest.raises(ValueError, match="cannot run EP rank 1"): + _frontend(ep_rank=1).bind_execution_state(state) + + +@pytest.mark.cpu_only +def test_bound_layer_passes_the_device_mask(monkeypatch: pytest.MonkeyPatch) -> None: + """Pass the bound state's device mask and abort state to the ops, also during CUDA graph capture.""" + calls = [] + payload = torch.zeros(_EP_SIZE, _TOKENS, _HIDDEN) + received = [payload, torch.zeros(_EP_SIZE, _TOKENS, 1, dtype=torch.int32)] + + def dispatch_op(*args, **kwargs): + calls.append(kwargs) + return received, 0, torch.empty(0) + + def combine_op(*args, **kwargs): + calls.append(kwargs) + return torch.zeros(_TOKENS, _HIDDEN) + + monkeypatch.setattr(torch.ops.trtllm, "moe_a2a_dispatch", dispatch_op, raising=False) + monkeypatch.setattr(torch.ops.trtllm, "moe_a2a_combine", combine_op, raising=False) + monkeypatch.setattr(torch.ops.trtllm, "moe_a2a_sanitize_expert_ids", Mock(), raising=False) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + state = _bindable_state( + ep_rank=0, + active_rank_mask=torch.zeros(4, dtype=torch.uint64), + abort=torch.zeros(4, dtype=torch.int32), + ) + hidden, experts = torch.zeros(_TOKENS, _HIDDEN), torch.zeros(_TOKENS, 1, dtype=torch.int32) + inputs = (hidden, None, experts, None, [_TOKENS] * _EP_SIZE) + without_mask = _frontend(fault_tolerance=False) + bound = _frontend() + with pytest.raises(RuntimeError, match="dispatches only once bound"): + bound.dispatch(*inputs) + bound.bind_execution_state(state) + for comm in (without_mask, bound): + comm.dispatch(*inputs) + comm.combine(payload) + + # Without fault tolerance the ops get no rank mask and no abort state. + fence = {"enable_rank_mask": False, "active_rank_mask": None, "use_sentinel": False} + assert calls[:2] == [ + {"expert_id_payload_index": None, "invalid_token_expert_id": None, **fence}, + {"use_low_precision": False, **fence}, + ] + for kwargs in calls[2:]: + assert kwargs["enable_rank_mask"] is True and kwargs["use_sentinel"] is False + assert kwargs["active_rank_mask"] is state.active_rank_mask + assert kwargs["abort_state"] is state.abort + state.register_layer.assert_called_once_with(bound.reset_state) + state.register_workspace.assert_called_once_with( + bound.mnnvl_mem, bound._reset_local_workspace, bound._check_incomplete_round + ) + + +def _moe_layer(cls: type, **attributes) -> torch.nn.Module: + """Build a MoE layer of class ``cls`` without its ``__init__`` and set ``attributes`` on it.""" + layer = cls.__new__(cls) + torch.nn.Module.__init__(layer) + for name, value in attributes.items(): + setattr(layer, name, value) + return layer + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "kind, uses", + [ + ("all_gather", "(ConfigurableMoE) uses AllGatherReduceScatter"), + ("no_communication", "(ConfigurableMoE) uses no communication strategy"), + ("triton", "(TritonFusedMoE) uses no communication strategy"), + ("vanilla", "(VanillaMoE) uses no communication strategy"), + ], +) +def test_unbindable_moe_layer_is_refused(kind: str, uses: str) -> None: + """Every MoE layer must bind, whatever its class.""" + layer = { + "all_gather": lambda: _moe_layer(ConfigurableMoE, comm=AllGatherReduceScatter(Mapping())), + "no_communication": lambda: _moe_layer(ConfigurableMoE, comm=None), + "triton": lambda: _moe_layer(TritonFusedMoE), + "vanilla": lambda: _moe_layer(VanillaMoE), + }[kind]() + bound = _moe_layer(ConfigurableMoE, comm=_frontend()) + with pytest.raises(ValueError, match=re.escape(f"MoE layer 1 {uses}")) as error: + moe_comms(torch.nn.ModuleList([bound, layer])) + assert "enable_wide_ep_fault_tolerance=False" in str(error.value) + + +@pytest.mark.cpu_only +def test_model_without_moe_layers_refused() -> None: + with pytest.raises(ValueError, match="requires a model with MoE layers"): + moe_comms(torch.nn.Linear(1, 1)) + + +@pytest.mark.cpu_only +def test_layers_bound_to_one_execution_state(monkeypatch: pytest.MonkeyPatch) -> None: + """prepare_model binds every MoE layer to one execution state over every EP rank.""" + monkeypatch.setattr( + rank_failure_model_load, + "MoeExecutionState", + lambda health, ep_rank, device: SimpleNamespace(ep_group_health=health, ep_rank=ep_rank), + ) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + comms = [Mock(spec=["bind_execution_state"]) for _ in range(2)] + model = torch.nn.ModuleList([_moe_layer(ConfigurableMoE, comm=comm) for comm in comms]) + model.model_config = SimpleNamespace(mapping=SimpleNamespace(moe_ep_size=4, moe_ep_rank=1)) + prepare_model(model) + (state,), (other,) = (comm.bind_execution_state.call_args.args for comm in comms) + assert state is other + assert state.ep_rank == 1 + assert state.ep_group_health.get_mask() == 0b1111 diff --git a/tests/unittest/_torch/moe/test_moe_impl.py b/tests/unittest/_torch/moe/test_moe_impl.py index c3e52ed33720..e8e395db442c 100644 --- a/tests/unittest/_torch/moe/test_moe_impl.py +++ b/tests/unittest/_torch/moe/test_moe_impl.py @@ -3515,6 +3515,7 @@ def test_cutlass_communication_selection(monkeypatch, is_oss, method, force_sour use_cuda_graph=False, use_low_precision_moe_combine=False, moe_load_balancer=None, + enable_wide_ep_fault_tolerance=False, ) constructors = {} for name in ( @@ -3527,7 +3528,6 @@ def test_cutlass_communication_selection(monkeypatch, is_oss, method, force_sour ): constructors[name] = MagicMock(name=name) monkeypatch.setattr(communication_factory, name, constructors[name]) - monkeypatch.setattr(communication_factory, "get_wide_ep_ft_options", lambda _: (None, 60, 1)) backend = TrtllmCutlassUnquantizedImpl.__new__(TrtllmCutlassUnquantizedImpl) # __new__ skips __init__, which normally sets quant_config from model_config. backend.quant_config = config.quant_config diff --git a/tests/unittest/_torch/moe/test_nvlink_one_sided_host.py b/tests/unittest/_torch/moe/test_nvlink_one_sided_host.py index a2ebc58a4676..ce010443d7df 100644 --- a/tests/unittest/_torch/moe/test_nvlink_one_sided_host.py +++ b/tests/unittest/_torch/moe/test_nvlink_one_sided_host.py @@ -128,6 +128,64 @@ def field(layout: torch.Tensor, name: str) -> int: ) +# The workspace regions after the round flag value, in the order calculateWorkspaceLayout places them. +_REGIONS_AFTER_FLAG_VAL = ( + "LOCAL_TOKEN_COUNTER", + "SEND_COUNTERS", + "RECV_COUNTERS", + "DISPATCH_COMPLETION_FLAGS", + "TOPK_TARGET_RANKS", + "TOPK_TARGET_INDICES", + "EPLB_GATHERED_STATS", + "DISPATCH_PAYLOAD", + "COMBINE_COMPLETION_FLAGS", + "COMBINE_INPUT", +) + + +@pytest.mark.parametrize( + "args", + [ + # ep_size, max_num_tokens_per_rank, top_k, dispatch_payload_bytes, combine_input_bytes, EPLB experts + (8, 128, 8, 1000, 3000, 288), + (36, 77, 6, 4096, 70000, None), + ], +) +def test_workspace_layout_without_rank_mask_has_no_incomplete_round_word(args: tuple) -> None: + """Only a rank-mask layout reserves the incomplete-round word, right after the round flag value.""" + from tensorrt_llm.bindings import internal as _tllm_internal + + thop = _tllm_internal.thop + + def offset(layout: torch.Tensor, region: str) -> int: + return int(layout[getattr(thop, f"MOE_A2A_{region}_OFFSET_INDEX")]) + + default = torch.ops.trtllm.moe_a2a_get_workspace_layout(*args) + without_mask = torch.ops.trtllm.moe_a2a_get_workspace_layout(*args, rank_mask=False) + with_mask = torch.ops.trtllm.moe_a2a_get_workspace_layout(*args, rank_mask=True) + assert torch.equal(without_mask, default) + assert offset(without_mask, "INCOMPLETE_ROUND") == 0 + + # The word follows the round flag value, and only the regions after it move. + assert offset(with_mask, "FLAG_VAL") == offset(without_mask, "FLAG_VAL") + assert offset(with_mask, "FLAG_VAL") < offset(with_mask, "INCOMPLETE_ROUND") + assert offset(with_mask, "INCOMPLETE_ROUND") < offset(with_mask, _REGIONS_AFTER_FLAG_VAL[0]) + for region in _REGIONS_AFTER_FLAG_VAL: + assert offset(with_mask, region) >= offset(without_mask, region) + unchanged = [ + int(getattr(thop, f"MOE_A2A_{name}_INDEX")) + for name in ( + "DISPATCH_PAYLOAD_SIZE", + "COMBINE_INPUT_SIZE", + "MAX_NUM_TOKENS", + "TOP_K", + "EP_SIZE", + "EPLB_STATS_NUM_EXPERTS", + ) + ] + assert torch.equal(with_mask[unchanged], without_mask[unchanged]) + + def test_sentinel_row_geometry() -> None: hidden_size, top_k = 7168, 8 # (hidden-state bytes, scale bytes, dispatch blocks per token) per row. diff --git a/tests/unittest/_torch/pyexecutor/test_error_classification.py b/tests/unittest/_torch/pyexecutor/test_error_classification.py index c8876b2c5c39..05ad7c95d5e0 100644 --- a/tests/unittest/_torch/pyexecutor/test_error_classification.py +++ b/tests/unittest/_torch/pyexecutor/test_error_classification.py @@ -62,7 +62,6 @@ def test_wide_ep_immediate_fatal_patterns(message): [ "NCCL timeout", "AlltoAll timeout waiting for completion_flags[3][7]", - "AlltoAll watchdog timed out waiting for peer rank 7", "completion_flags timeout", "NCCL operation timed out during all_reduce", "deep_ep buffer barrier hang", diff --git a/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py b/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py index dcbd5cc835f0..cc324e95bbb5 100644 --- a/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py +++ b/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py @@ -82,36 +82,18 @@ def _make_lifecycle() -> tuple[_MnnvlAlltoAllWorkspaceLifecycle, Mock, torch.Ten memory=memory, workspace=workspace, metainfo=metainfo, - metainfo_index={ - "FLAG_VAL_OFFSET_INDEX": 0, - "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX": 0, - "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX": 0, - }, ep_rank=0, ep_size=2, - health=None, ) return lifecycle, memory, metainfo -def _register_without_watchdog( - lifecycle: _MnnvlAlltoAllWorkspaceLifecycle, - client: _Client, -) -> None: - lifecycle.register( - client, - watchdog_timeout_s=None, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - def test_checkpoint_prepare_rejects_any_active_shared_client() -> None: lifecycle, memory, _ = _make_lifecycle() idle = _Client() - _register_without_watchdog(lifecycle, idle) + lifecycle.register(idle) active = _Client(idle=False) - _register_without_watchdog(lifecycle, active) + lifecycle.register(active) with pytest.raises(RuntimeError, match="active MoE All-to-All phase"): lifecycle.checkpoint_prepare() @@ -129,27 +111,6 @@ def test_repeated_checkpoint_prepare_skips_shared_preflight() -> None: memory.checkpoint_prepare.assert_called_once_with() -def test_detached_checkpoint_prepare_stops_stale_watchdog() -> None: - lifecycle, memory, _ = _make_lifecycle() - coordinator = Mock() - watchdog = Mock() - coordinator.acquire_watchdog.return_value = watchdog - lifecycle._coordinator = coordinator - lifecycle.register( - _Client(), - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - memory.mapped = False - - lifecycle.checkpoint_prepare() - - coordinator.release_watchdog.assert_called_once_with(watchdog) - assert memory.comm.allgather_count == 0 - memory.checkpoint_prepare.assert_called_once_with() - - def test_checkpoint_prepare_rejects_uninitialized_communicator() -> None: lifecycle, memory, _ = _make_lifecycle() memory.comm = None @@ -193,180 +154,49 @@ def test_checkpoint_prepare_timeout_fails_closed(monkeypatch) -> None: memory.checkpoint_prepare.assert_not_called() -def test_checkpoint_prepare_rejects_remote_active_client_before_watchdog_stop() -> None: +def test_checkpoint_prepare_rejects_remote_active_client() -> None: lifecycle, memory, _ = _make_lifecycle() memory.comm = _FakeComm(clients_idle_by_rank=[True, False]) - coordinator = Mock() - watchdog = Mock() - coordinator.acquire_watchdog.return_value = watchdog - lifecycle._coordinator = coordinator - client = _Client() - lifecycle.register( - client, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) + lifecycle.register(_Client()) with pytest.raises(RuntimeError, match=r"active MoE All-to-All phase on ranks \[1\]"): lifecycle.checkpoint_prepare() - coordinator.release_watchdog.assert_not_called() memory.checkpoint_prepare.assert_not_called() -def test_checkpoint_suspends_and_recreates_one_shared_watchdog( +def test_checkpoint_restore_resets_every_shared_client( monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, memory, metainfo = _make_lifecycle() - old_coordinator = Mock() - old_watchdog = Mock() - old_coordinator.acquire_watchdog.return_value = old_watchdog - lifecycle._coordinator = old_coordinator first = _Client() second = _Client() - for client in (first, second): - lifecycle.register( - client, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - old_coordinator.acquire_watchdog.assert_called_once_with( - ep_size=2, - timeout_s=5.0, - poll_interval_s=0.1, - on_timeout=None, - ) - assert lifecycle.watchdog_for(first) is old_watchdog - assert lifecycle.watchdog_for(second) is old_watchdog + lifecycle.register(client) lifecycle.checkpoint_prepare() - old_coordinator.release_watchdog.assert_called_once_with(old_watchdog) memory.checkpoint_prepare.assert_called_once_with() memory.mapped = False memory.checkpoint_restore.return_value = True - new_coordinator = Mock() - new_watchdog = Mock() - new_coordinator.acquire_watchdog.return_value = new_watchdog - monkeypatch.setattr( - lifecycle, - "_create_coordinator", - Mock(return_value=new_coordinator), - ) monkeypatch.setattr(torch.cuda, "synchronize", Mock()) comm = _FakeComm() lifecycle.checkpoint_restore(comm, Mock(return_value=metainfo)) memory._checkpoint_restore_complete.assert_called_once_with() - new_coordinator.acquire_watchdog.assert_called_once_with( - ep_size=2, - timeout_s=5.0, - poll_interval_s=0.1, - on_timeout=None, - ) - assert lifecycle.watchdog_for(first) is new_watchdog - assert lifecycle.watchdog_for(second) is new_watchdog assert first.reset_count == 1 assert second.reset_count == 1 assert comm.allgather_count == 1 -def test_unregister_stops_watchdog_after_last_enabled_client() -> None: - lifecycle, _, _ = _make_lifecycle() - coordinator = Mock() - watchdog = Mock() - coordinator.acquire_watchdog.return_value = watchdog - lifecycle._coordinator = coordinator - first = _Client() - second = _Client() - for client in (first, second): - lifecycle.register( - client, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - lifecycle.unregister(first) - coordinator.release_watchdog.assert_not_called() - - lifecycle.unregister(second) - coordinator.release_watchdog.assert_called_once_with(watchdog) - - -def test_shared_watchdog_configuration_mismatch_rejects_new_client() -> None: - lifecycle, _, _ = _make_lifecycle() - coordinator = Mock() - coordinator.acquire_watchdog.return_value = Mock() - lifecycle._coordinator = coordinator - first = _Client() - second = _Client() - lifecycle.register( - first, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - with pytest.raises(ValueError, match="same watchdog configuration"): - lifecycle.register( - second, - watchdog_timeout_s=10.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - assert lifecycle.watchdog_for(second) is None - coordinator.acquire_watchdog.assert_called_once() - - -def test_watchdog_registration_is_deferred_while_workspace_is_unmapped( - monkeypatch: pytest.MonkeyPatch, -) -> None: - lifecycle, memory, metainfo = _make_lifecycle() - memory.mapped = False - memory.checkpoint_restore.return_value = True - old_coordinator = Mock() - lifecycle._coordinator = old_coordinator - client = _Client() - - lifecycle.register( - client, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) - - old_coordinator.acquire_watchdog.assert_not_called() - assert lifecycle.watchdog_for(client) is None - - new_coordinator = Mock() - new_watchdog = Mock() - new_coordinator.acquire_watchdog.return_value = new_watchdog - monkeypatch.setattr( - lifecycle, - "_create_coordinator", - Mock(return_value=new_coordinator), - ) - monkeypatch.setattr(torch.cuda, "synchronize", Mock()) - - lifecycle.checkpoint_restore(_FakeComm(), Mock(return_value=metainfo)) - - assert lifecycle.watchdog_for(client) is new_watchdog - - -def test_checkpoint_restore_failure_before_watchdog_start_stays_unpublished( +def test_checkpoint_restore_failure_stays_unpublished( monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, memory, _ = _make_lifecycle() client = _Client() - _register_without_watchdog(lifecycle, client) + lifecycle.register(client) memory.mapped = False memory.checkpoint_restore.return_value = True monkeypatch.setattr(torch.cuda, "synchronize", Mock()) @@ -387,7 +217,7 @@ def test_checkpoint_restore_rejects_changed_metainfo_and_fails_closed( ) -> None: lifecycle, memory, _ = _make_lifecycle() client = _Client() - _register_without_watchdog(lifecycle, client) + lifecycle.register(client) memory.mapped = False memory.checkpoint_restore.return_value = True monkeypatch.setattr(torch.cuda, "synchronize", Mock()) @@ -403,38 +233,21 @@ def test_checkpoint_restore_rejects_changed_metainfo_and_fails_closed( assert client.reset_count == 0 -def test_checkpoint_restore_failure_after_watchdog_start_stops_it( +def test_checkpoint_restore_client_reset_failure_fails_closed( monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle, memory, metainfo = _make_lifecycle() - old_coordinator = Mock() - old_watchdog = Mock() - old_coordinator.acquire_watchdog.return_value = old_watchdog - lifecycle._coordinator = old_coordinator + # The lifecycle holds its clients weakly, so the test keeps this one alive. client = _FailingResetClient() - lifecycle.register( - client, - watchdog_timeout_s=5.0, - watchdog_poll_interval_s=0.1, - watchdog_on_timeout=None, - ) + lifecycle.register(client) lifecycle.checkpoint_prepare() memory.mapped = False memory.checkpoint_restore.return_value = True - new_coordinator = Mock() - new_watchdog = Mock() - new_coordinator.acquire_watchdog.return_value = new_watchdog - monkeypatch.setattr( - lifecycle, - "_create_coordinator", - Mock(return_value=new_coordinator), - ) monkeypatch.setattr(torch.cuda, "synchronize", Mock()) with pytest.raises(RuntimeError, match="frontend reset failed"): lifecycle.checkpoint_restore(_FakeComm(), Mock(return_value=metainfo)) - new_coordinator.release_watchdog.assert_called_once_with(new_watchdog) memory._checkpoint_restore_failed.assert_called_once_with() memory._checkpoint_restore_complete.assert_not_called() @@ -444,7 +257,7 @@ def test_checkpoint_restore_remote_failure_fails_closed_on_every_rank( ) -> None: lifecycle, memory, metainfo = _make_lifecycle() client = _Client() - _register_without_watchdog(lifecycle, client) + lifecycle.register(client) memory.mapped = False memory.checkpoint_restore.return_value = True monkeypatch.setattr(torch.cuda, "synchronize", Mock()) @@ -793,9 +606,6 @@ def as_torch_strided_tensor(self, dtype: torch.dtype) -> torch.Tensor: monkeypatch.setattr(NVLinkOneSided, "_WORKSPACE", None) monkeypatch.setattr(NVLinkOneSided, "is_platform_supported", Mock(return_value=True)) monkeypatch.setattr(NVLinkOneSided, "_init_constants", Mock()) - monkeypatch.setattr(NVLinkOneSided, "FLAG_VAL_OFFSET_INDEX", 0) - monkeypatch.setattr(NVLinkOneSided, "DISPATCH_COMPLETION_FLAGS_OFFSET_INDEX", 0) - monkeypatch.setattr(NVLinkOneSided, "COMBINE_COMPLETION_FLAGS_OFFSET_INDEX", 0) monkeypatch.setattr(one_sided_module, "MnnvlMemory", _Memory) monkeypatch.setattr( _MnnvlAlltoAllWorkspaceLifecycle, diff --git a/tests/unittest/api_stability/references/llm.yaml b/tests/unittest/api_stability/references/llm.yaml index 5511ed0d1b55..11b5897e2c80 100644 --- a/tests/unittest/api_stability/references/llm.yaml +++ b/tests/unittest/api_stability/references/llm.yaml @@ -390,6 +390,10 @@ methods: annotation: Optional[tensorrt_llm.llmapi.llm_args.SleepConfig] default: null status: prototype + enable_wide_ep_fault_tolerance: + annotation: bool + default: False + status: prototype reorder_policy_config: annotation: Optional[tensorrt_llm.llmapi.llm_args.ReorderRequestPolicyConfig] default: null diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 6f693a18f819..90cfffa465d0 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -155,6 +155,14 @@ def test_checkpoint_io_policy_rejects_unknown_value() -> None: TorchLlmArgs(model=llama_model_path, checkpoint_io_policy="unknown") +@pytest.mark.cpu_only +def test_wide_ep_fault_tolerance_defaults_off( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TLLM_FAULT_TOLERANCE_MODE", raising=False) + assert not TorchLlmArgs( + model=llama_model_path).enable_wide_ep_fault_tolerance + + @pytest.mark.cpu_only @pytest.mark.parametrize( ("model_kwargs", "expected"), diff --git a/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py b/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py index 5a3807a3bb7f..260814b2c026 100644 --- a/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py +++ b/tests/unittest/llmapi/test_trtllm_serve_config_schemas.py @@ -73,6 +73,8 @@ def schemas(tmp_path_factory: pytest.TempPathFactory) -> dict[str, dict]: }, True, ), + (generator.SERVE_SCHEMA, {"enable_wide_ep_fault_tolerance": True}, True), + (generator.SERVE_SCHEMA, {"enable_wide_ep_fault_tolerance": "on"}, False), (generator.SERVE_SCHEMA, {"moe_config": {"backend": "CUTLASS"}}, True), (generator.SERVE_SCHEMA, {"moe_config": {"backend_typo": "CUTLASS"}}, False), (generator.SERVE_SCHEMA, {"moe_config": {"backend": "not-a-backend"}}, False), From 72315d52651fc5d3da83234602225346c196986d Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Wed, 18 Nov 2026 21:28:19 -0800 Subject: [PATCH 02/13] [None][test] Set enable_wide_ep_fault_tolerance in the model loader tests' llm_args ModelLoader reads llm_args.enable_wide_ep_fault_tolerance when it builds the model, so the SimpleNamespace llm_args of the MX and GMS loader tests set it like their other fields. Signed-off-by: Andrew Rusli --- tests/unittest/_torch/pyexecutor/test_model_loader_gms.py | 1 + tests/unittest/_torch/pyexecutor/test_model_loader_mx.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py index 04febf65f0d0..a16ba19f8973 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py +++ b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py @@ -105,6 +105,7 @@ def _make_loader(monkeypatch, *, events, spec_config=None): llm_args = SimpleNamespace( load_format=LoadFormat.GMS, lm_head_output_dtype="auto", + enable_wide_ep_fault_tolerance=False, gms_config=SimpleNamespace(socket_path="/tmp/gms.sock", mode="auto", tag="weights"), ) loader = ModelLoader( diff --git a/tests/unittest/_torch/pyexecutor/test_model_loader_mx.py b/tests/unittest/_torch/pyexecutor/test_model_loader_mx.py index 679776cc7749..b9f123ecc94a 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_loader_mx.py +++ b/tests/unittest/_torch/pyexecutor/test_model_loader_mx.py @@ -1130,7 +1130,11 @@ def _tiny_profile_registry(*, speculative_mode: str | None = None) -> PostTransf def _make_loader(monkeypatch, *, events, spec_config=None): - llm_args = SimpleNamespace(load_format=LoadFormat.AUTO, lm_head_output_dtype="auto") + llm_args = SimpleNamespace( + load_format=LoadFormat.AUTO, + lm_head_output_dtype="auto", + enable_wide_ep_fault_tolerance=False, + ) loader = ModelLoader( llm_args=llm_args, mapping=MagicMock(name="mapping"), From 93aceb421d4e25a218b8d44ca8fa87f24a2a8a60 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Thu, 19 Nov 2026 09:56:35 -0800 Subject: [PATCH 03/13] [None][fix] Address the review of the fault-tolerant MoE all-to-all Bind each MoE layer with the EP rank and size of its own communication, and refuse layers that disagree, drafter layers included, and execution states of another EP size. Reject the configurations the arguments alone rule out at validation, use the built-in model under modeling_v2 auto, and name the actual cause when a layer cannot be bound. Forward the field to drafter model configs. Keep an abort request that arrives during rearm, refuse a mask install while a layer is between its dispatch and its combine, and document the recovery order. Check the abort state's pinning once, at construction, and refuse Confidential Computing. Warn at every MoE communication selection when a removed all-to-all watchdog setting or TLLM_FAULT_TOLERANCE_MODE=1 is set. Describe the field's prerequisites, fix stale comments and coding-guideline nits, bound the barriers of the multi-GPU tests, and test the fence timeout without an abort state, the low-precision combine of an aborted round and an aborted combine. Signed-off-by: Andrew Rusli --- .../moe/communication/moeAlltoAllKernels.cu | 2 + .../moe/communication/moeAlltoAllKernels.h | 18 +- .../thop/moe/communication/moeAlltoAllMeta.h | 8 +- .../thop/moe/communication/moeAlltoAllOp.cpp | 26 +- .../_torch/models/modeling_speculative.py | 14 +- .../communication/communication_factory.py | 27 ++ .../communication/execution_state.py | 78 ++++-- .../communication/nvlink_one_sided.py | 16 +- .../_torch/moe/fused_moe/ep_group_health.py | 8 +- .../_torch/pyexecutor/model_loader.py | 2 - .../pyexecutor/rank_failure_model_load.py | 56 ++++- tensorrt_llm/llmapi/llm_args.py | 61 ++++- .../schemas/trtllm-serve-config.schema.json | 2 +- .../trtllm-serve-disagg-config.schema.json | 4 +- .../modeling/test_modeling_speculative.py | 63 +++++ .../unittest/_torch/moe/moe_a2a_test_utils.py | 20 +- .../moe/multi_gpu/test_nvlink_one_sided.py | 58 +++-- .../_torch/moe/test_communication_factory.py | 56 ++++- .../unittest/_torch/moe/test_moe_a2a_abort.py | 97 ++++++-- .../_torch/moe/test_moe_execution_state.py | 235 +++++++++++++++--- .../pyexecutor/test_model_loader_gms.py | 27 ++ .../_torch/test_mnnvl_alltoall_workspace.py | 3 +- tests/unittest/llmapi/test_llm_args.py | 97 +++++++- 23 files changed, 822 insertions(+), 156 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu index d8b51527e329..52209a7c76a8 100644 --- a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.cu @@ -1268,7 +1268,9 @@ __device__ __forceinline__ void prepareCombine(uint8_t* combine_input_base, void if constexpr (HAS_INCOMPLETE_ROUND) { if (rank_idx != ep_rank && only_own_rows_valid(*incomplete_round, flag_value)) + { return; + } } size_t const source_offset = static_cast(global_token_idx) * source_stride_per_token; diff --git a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h index 08b437ad4eb0..5e3d5618b632 100644 --- a/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h +++ b/cpp/tensorrt_llm/kernels/moe/communication/moeAlltoAllKernels.h @@ -40,11 +40,11 @@ static constexpr int64_t kDefaultTimeoutCycles = 300ll * kAssumedClockHz; // Waits on a peer shorter than kAbortPollCycles never read the host abort state. // Longer waits read it once per kAbortPollCycles per block. -static constexpr int64_t kAbortPollCycles = 100ll * 1000ll; +static constexpr int64_t kAbortPollCycles = 100LL * 1000LL; // A wait on a peer longer than kPublishWaitedFlagCycles publishes the flag value it waits for in // MoeA2AAbortState::waited_flag, so the host can tell which peer this rank waits on. -static constexpr int64_t kPublishWaitedFlagCycles = 20ll * 1000ll * 1000ll; +static constexpr int64_t kPublishWaitedFlagCycles = 20LL * 1000LL * 1000LL; //! \brief Abort state of the rank-mask collectives, in pinned host memory. //! @@ -56,10 +56,10 @@ struct MoeA2AAbortState static constexpr int32_t kCancelled = 1; int32_t requested; - // kCancelled once a kernel stopped waiting because of `requested`, or the incomplete-round check found the word - // set, 0 otherwise + //! kCancelled once a kernel stopped waiting because of `requested`, or the incomplete-round check found the word + //! set, 0 otherwise int32_t status; - // Per peer, the round flag value that a long wait on the peer waits for, 0 otherwise + //! Per peer, the round flag value that a long wait on the peer waits for, 0 otherwise uint32_t waited_flag[kMaxRanks]; }; @@ -294,16 +294,16 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params); void moe_a2a_prepare_combine_launch(MoeA2ACombineParams const& params); -// Mark `abort` cancelled if the incomplete-round word (see above) is set, ordered after every earlier fence combine in -// `stream`. Launch once per forward, after its last combine. +//! Mark `abort` cancelled if the incomplete-round word (see above) is set, ordered after every earlier fence combine +//! in `stream`. Launch once per forward, after its last combine. void moe_a2a_check_incomplete_round_launch( uint32_t const* incomplete_round, MoeA2AAbortState* abort, cudaStream_t stream); // Sanitize expert IDs for invalid tokens // expert_ids: [ep_size, max_tokens_per_rank, top_k] (int32) // recv_counters: [2, ep_size] (int32), number of valid tokens per source -// incomplete_round: incomplete-round word (see above), or nullptr. While it is nonzero, only the tokens from ep_rank -// are valid. +// incomplete_round: incomplete-round word (see above), or nullptr. While it holds a nonzero value other than the +// incomplete-round value of this round's combine, only the tokens from ep_rank are valid. // invalid_id: value to fill for invalid tokens' expert ids void moe_a2a_sanitize_expert_ids_launch(int32_t* expert_ids, int32_t const* recv_counters, uint32_t const* flag_val, uint32_t const* incomplete_round, int32_t invalid_id, int ep_rank, int ep_size, int max_tokens_per_rank, int top_k, diff --git a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h index 669f7914665f..e43a758f8c68 100644 --- a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h +++ b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllMeta.h @@ -112,6 +112,12 @@ enum MoeA2AMetaInfoIndex : int64_t using MoeA2AWorkspaceLayout = std::array; static constexpr int64_t kWorkspaceAlignment = 256; +// The host holds the abort state as an int32 tensor of kAbortStateWords words and addresses `requested` and `status` +// as its words 0 and 1. +static constexpr int64_t kAbortStateWords = sizeof(tensorrt_llm::kernels::moe_comm::MoeA2AAbortState) / sizeof(int32_t); +static_assert(offsetof(tensorrt_llm::kernels::moe_comm::MoeA2AAbortState, requested) == 0); +static_assert(offsetof(tensorrt_llm::kernels::moe_comm::MoeA2AAbortState, status) == sizeof(int32_t)); + inline std::vector> getMoeA2AMetaInfoIndexPairs() { using namespace tensorrt_llm::kernels::moe_comm; @@ -156,7 +162,7 @@ inline std::vector> getMoeA2AMetaInfoIndexPairs( {"MOE_A2A_SENTINEL_FORMAT_MXFP8", static_cast(sentinel::PayloadFormat::kMxfp8)}, {"MOE_A2A_SENTINEL_FORMAT_FP8_BLOCK128", static_cast(sentinel::PayloadFormat::kFp8Block128)}, {"MOE_A2A_SENTINEL_FORMAT_NVFP4", static_cast(sentinel::PayloadFormat::kNvfp4)}, - {"MOE_A2A_ABORT_STATE_WORDS", sizeof(MoeA2AAbortState) / sizeof(int32_t)}, + {"MOE_A2A_ABORT_STATE_WORDS", kAbortStateWords}, {"MOE_A2A_ABORT_WAITED_FLAG_INDEX", offsetof(MoeA2AAbortState, waited_flag) / sizeof(int32_t)}, }; } diff --git a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp index 6799fa19b278..4126dae5d42b 100644 --- a/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moe/communication/moeAlltoAllOp.cpp @@ -127,12 +127,9 @@ inline bool hasActiveRankMask(torch::optional const& maskTensor) // Return the device address of a provided rank-mask tensor, which the kernels read when they run. On failure // (wrong dtype / device / shape), throw at the Python op boundary rather than launch. inline uint64_t const* resolveActiveRankMask( - torch::optional const& maskTensor, int64_t epRank, torch::Tensor const& workspace) + torch::optional const& maskTensor, torch::Tensor const& workspace) { using tensorrt_llm::kernels::moe_comm::kRankMaskWords; - using tensorrt_llm::kernels::moe_comm::kMaxRanks; - TORCH_CHECK( - epRank >= 0 && epRank < kMaxRanks, "epRank must be in the range [0, ", kMaxRanks, ") for active_rank_mask"); TORCH_CHECK(hasActiveRankMask(maskTensor), "active_rank_mask must be defined"); torch::Tensor const& t = maskTensor.value(); TORCH_CHECK(t.device() == workspace.device(), "active_rank_mask must be on the workspace device"); @@ -144,7 +141,7 @@ inline uint64_t const* resolveActiveRankMask( } // The kernels read the host abort state through unified virtual addressing, so the caller allocates it in pinned -// memory. +// memory. MoeExecutionState checks that once, when it allocates the abort state. inline tensorrt_llm::kernels::moe_comm::MoeA2AAbortState* resolveAbortState(torch::optional const& abort) { using tensorrt_llm::kernels::moe_comm::MoeA2AAbortState; @@ -153,11 +150,9 @@ inline tensorrt_llm::kernels::moe_comm::MoeA2AAbortState* resolveAbortState(torc return nullptr; } torch::Tensor const& t = abort.value(); - constexpr int64_t kWords = sizeof(MoeA2AAbortState) / sizeof(int32_t); - TORCH_CHECK( - t.is_cpu() && t.scalar_type() == torch::kInt32 && t.dim() == 1 && t.numel() == kWords && t.is_contiguous(), - "abort_state must be a contiguous int32 CPU tensor of ", kWords, " elements"); - TORCH_CHECK(t.is_pinned(), "abort_state must be in pinned memory, because the kernels read it at its host address"); + TORCH_CHECK(t.is_cpu() && t.scalar_type() == torch::kInt32 && t.dim() == 1 && t.numel() == kAbortStateWords + && t.is_contiguous(), + "abort_state must be a contiguous int32 CPU tensor of ", kAbortStateWords, " elements"); return static_cast(t.data_ptr()); } @@ -704,8 +699,7 @@ void sentinelCombine(torch::Tensor const& payload, int64_t localNumTokens, torch // - numExperts: Total number of routing slots (tokenSelectedExperts values are in [0, numExperts)) // - eplbStatsNumExperts: Number of experts used for EPLB stats (may be <= numExperts) // - eplbLocalStats: [eplbStatsNumExperts] tensor containing local statistics for EPLB. -// - enableRankMask: Exchange only with the ranks set in activeRankMask. The workspace layout must be built with -// rank_mask=True. +// - enableRankMask: Exchange only with the ranks set in activeRankMask. // - activeRankMask: [kRankMaskWords] uint64 tensor on the workspace device. The kernels read it at execution time, // so a captured CUDA graph follows later updates. The kernels treat the local rank's bit as set. // - useSentinel: run the sentinel transport; the payloads must be the hidden states, their scales (quantized @@ -713,8 +707,8 @@ void sentinelCombine(torch::Tensor const& payload, int64_t localNumTokens, torch // expertIdPayloadIndex / invalidTokenExpertId must be given. Received rows are then (source rank, token), not // compacted, and rows of tokens not routed here carry invalidTokenExpertId. // - abortState: (Optional) pinned int32 host tensor holding a MoeA2AAbortState, for enableRankMask without -// useSentinel. Long waits on peers publish the flag value they wait for in it. Setting its requested word stops the -// waits on peers. +// useSentinel, with a workspace layout built with rank_mask=True. Long waits on peers publish the flag value they +// wait for in it. Setting its requested word stops the waits on peers. // // Return values: // - recvTensors: Vector of receive buffers (one tensor per payload), each [ep_size, runtimeMaxTokensPerRank, @@ -973,7 +967,7 @@ std::tuple, int64_t, torch::Tensor> moeA2ADispatchOp( params.enable_rank_mask = enableRankMask; if (params.enable_rank_mask) { - params.active_rank_mask = resolveActiveRankMask(activeRankMask, epRank, workspace); + params.active_rank_mask = resolveActiveRankMask(activeRankMask, workspace); } else { @@ -1161,7 +1155,7 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke params.enable_rank_mask = enableRankMask; if (params.enable_rank_mask) { - params.active_rank_mask = resolveActiveRankMask(activeRankMask, epRank, workspace); + params.active_rank_mask = resolveActiveRankMask(activeRankMask, workspace); } else { diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index 6a164dbee671..a3d218cb705a 100644 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -1841,6 +1841,10 @@ def external_drafter_config_kwargs(model_config, spec_config) -> dict: # the same format as the target's. use_low_precision_moe_combine=model_config. use_low_precision_moe_combine, + # So is Wide-EP fault tolerance: the drafter's MoE layers bind to the + # target's execution state. + enable_wide_ep_fault_tolerance=model_config. + enable_wide_ep_fault_tolerance, # A MEGAMOE_CUTEDSL drafter shares the target's bucket workspaces, so its kernels must match. megamoe_fuse_shared_expert=model_config.megamoe_fuse_shared_expert, ) @@ -2126,6 +2130,8 @@ def __init__(self, moe_max_num_tokens=model_config.moe_max_num_tokens, max_num_tokens=model_config.max_num_tokens, moe_load_balancer=model_config.moe_load_balancer, + enable_wide_ep_fault_tolerance=model_config. + enable_wide_ep_fault_tolerance, skip_create_weights_in_init=True, ) if draft_moe_selection is not None: @@ -2147,7 +2153,9 @@ def __init__(self, ), spec_config=model_config.spec_config, max_num_tokens=model_config.max_num_tokens, - moe_max_num_tokens=model_config.moe_max_num_tokens) + moe_max_num_tokens=model_config.moe_max_num_tokens, + enable_wide_ep_fault_tolerance=model_config. + enable_wide_ep_fault_tolerance) else: raise ValueError( f"Unsupported eagle3 model architecture for draft model: {spec_config.eagle3_model_arch}" @@ -2169,7 +2177,9 @@ def __init__(self, mapping=model_config.mapping.mapping_for_full_rows(), spec_config=None, max_num_tokens=model_config.max_num_tokens, - moe_max_num_tokens=model_config.moe_max_num_tokens) + moe_max_num_tokens=model_config.moe_max_num_tokens, + enable_wide_ep_fault_tolerance=model_config. + enable_wide_ep_fault_tolerance) self.draft_config.extra_attrs = model_config.extra_attrs _set_draft_kv_cache_quant_algo(self.draft_config, model_config) diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py b/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py index 9a9476b25df0..cc35b8cd8acf 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/communication_factory.py @@ -45,6 +45,32 @@ if TYPE_CHECKING: from tensorrt_llm.models.modeling_utils import QuantConfig +# Settings of the removed all-to-all watchdog, which no MoE communication reads. +_REMOVED_WATCHDOG_ENVS = ( + "TRTLLM_ALLTOALL_WATCHDOG_TIMEOUT_S", + "TRTLLM_ALLTOALL_WATCHDOG_POLL_INTERVAL_S", +) +_FAULT_TOLERANCE_MODE_ENV = "TLLM_FAULT_TOLERANCE_MODE" + + +def _warn_on_removed_fault_tolerance_envs() -> None: + """Warn once about each set variable of the earlier Wide-EP fault tolerance mode.""" + for name in _REMOVED_WATCHDOG_ENVS: + if name in os.environ: + logger.warning_once( + f"{name} is no longer read: the all-to-all watchdog was removed. " + "Use enable_wide_ep_fault_tolerance for Wide-EP fault tolerance.", + key=f"removed_env_{name}", + ) + if os.environ.get(_FAULT_TOLERANCE_MODE_ENV) == "1": + logger.warning_once( + f"{_FAULT_TOLERANCE_MODE_ENV}=1 no longer turns on Wide-EP fault tolerance for the " + "MoE layers; set enable_wide_ep_fault_tolerance=True. It only selects the MPI " + "signal-handler mode, which is deprecated and will be removed in a future release.", + key=f"removed_env_{_FAULT_TOLERANCE_MODE_ENV}", + ) + + # Temporary NCCL-EP v0.1.0 limitation. The LL combine kernel derives its # warp-group count and dynamic-SMEM requirement here: # https://github.com/NVIDIA/nccl/blob/nccl-ep-v0.1.0/contrib/nccl_ep/device/low_latency.cu#L1990-L2025 @@ -142,6 +168,7 @@ def create_strategy( # Backend constructors apply these budgets, and auto-selection below treats a constructor # error as "backend unavailable", so an invalid variable must fail here instead. get_moe_comm_timeout_budgets() + _warn_on_removed_fault_tolerance_envs() # Extract parameters from model_config mapping = model_config.mapping diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py b/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py index 1021ee996069..7f63f8c42ec8 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/execution_state.py @@ -15,25 +15,28 @@ """The active-rank mask and abort state that the MoE all-to-all kernels read under Wide-EP fault tolerance. When a peer stops, ``request_abort`` releases the collectives that wait for it. Then, with the model -idle, ``rearm`` returns the bound workspaces and layers to their first round, and ``install_mask`` -writes the new rank mask. Long waits on peers publish the round flag they wait for in the abort -state (``waited_flags``). After each forward, ``check_incomplete_rounds`` marks the abort state -cancelled if a peer may have overwritten rows that the forward's combines read. +idle on every surviving rank, ``rearm`` returns the bound workspaces and layers to their first round, +and ``install_mask`` writes the new rank mask (see ``rearm`` for the required order). Long waits on +peers publish the round flag they wait for in the abort state (``waited_flags``). After each +forward, ``check_incomplete_rounds`` marks the abort state cancelled if a peer may have overwritten +rows that the forward's combines read. """ from __future__ import annotations +import threading from collections.abc import Callable import torch -from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm._utils import confidential_compute_enabled, prefer_pinned from tensorrt_llm.bindings import internal as _tllm_internal from ..ep_group_health import EP_MASK_NUM_WORDS, EPGroupHealth _MASK_WORD_BITS = 64 _ABORT_REQUESTED = 0 +_ABORT_STATUS = 1 # The words ``requested`` and ``status``, then the round flag awaited from each peer. _ABORT_WORDS = slice(0, int(_tllm_internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX)) _WAITED_FLAGS = slice(int(_tllm_internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX), None) @@ -65,6 +68,11 @@ def __init__(self, ep_group_health: EPGroupHealth, ep_rank: int, device: torch.d self.ep_rank = ep_rank self._device = device # The kernels read the abort state at its host address, which must be pinned memory. + if confidential_compute_enabled(): + raise RuntimeError( + "Wide-EP fault tolerance is not supported with Confidential Computing: the MoE " + "All-to-All kernels read their abort state in pinned host memory" + ) self.abort = torch.zeros( int(_tllm_internal.thop.MOE_A2A_ABORT_STATE_WORDS), dtype=torch.int32, @@ -73,17 +81,25 @@ def __init__(self, ep_group_health: EPGroupHealth, ep_rank: int, device: torch.d if not self.abort.is_pinned(): raise RuntimeError("The MoE All-to-All abort state requires pinned host memory") self._abort_words = self.abort.numpy()[_ABORT_WORDS] + # Orders the updates of ``requested`` by request_abort and rearm. + self._abort_lock = threading.Lock() self._workspaces: dict[object, tuple[Callable[[], None], Callable[[], None]]] = {} - self._layer_resets: list[Callable[[], None]] = [] + self._layers: list[tuple[Callable[[], None], Callable[[], bool]]] = [] snapshot = ep_group_health.snapshot() self._generation = snapshot.generation self.active_rank_mask = torch.tensor( _mask_words(snapshot.mask, ep_rank), dtype=torch.uint64, device=device ) - def register_layer(self, reset_state: Callable[[], None]) -> None: - """Register the reset of a bound layer's dispatch/combine state machine.""" - self._layer_resets.append(reset_state) + def register_layer( + self, reset_state: Callable[[], None], between_rounds: Callable[[], bool] + ) -> None: + """Register a bound layer's dispatch/combine state machine. + + ``reset_state`` returns it to idle, and ``between_rounds`` returns whether it is idle, that + is, no dispatch of the layer waits for its combine. + """ + self._layers.append((reset_state, between_rounds)) def register_workspace( self, @@ -99,8 +115,12 @@ def register_workspace( self._workspaces.setdefault(workspace, (reset_workspace, check_incomplete_round)) def request_abort(self) -> None: - """Make running and future collectives return without their peers. Safe from any thread.""" - self._abort_words[_ABORT_REQUESTED] = 1 + """Make running and future collectives return without their peers. Safe from any thread. + + The request stays in effect until a :meth:`rearm` that starts after it. + """ + with self._abort_lock: + self._abort_words[_ABORT_REQUESTED] += 1 def aborted(self) -> bool: """Return whether an abort was requested or a collective aborted since the last :meth:`rearm`.""" @@ -129,28 +149,52 @@ def waited_flags(self) -> dict[int, int]: def rearm(self) -> None: """Clear the abort state and return every bound workspace and layer to its first round. - Call only while the model is idle and before :meth:`install_mask`. The device is - synchronized before and after, so no queued collective sees a partial reset. + It resets this rank's row of each bound workspace without a barrier, so every surviving + rank recovers in this order: :meth:`request_abort` if a collective waits on a failed peer, + a device synchronization, a barrier over the surviving ranks, ``rearm``, + :meth:`install_mask`, then a barrier over the surviving ranks before its next collective. + The first barrier keeps a peer's aborted collectives from writing into the row after its + reset, the second keeps a peer's next collective from writing into it before. ``install_mask`` + needs no barrier of its own: it writes only this rank's device mask, after a device + synchronization, and refuses a layer that is between its dispatch and its combine. The + device is synchronized before and after, so no queued collective sees a partial reset. + + The abort requests made before ``rearm`` starts are cleared. A request made meanwhile + stays in effect, so :meth:`aborted` then holds and :meth:`install_mask` raises. """ + with self._abort_lock: + handled_requests = int(self._abort_words[_ABORT_REQUESTED]) torch.cuda.synchronize(self._device) - self.abort.zero_() for reset_workspace, _ in self._workspaces.values(): reset_workspace() - for reset_state in self._layer_resets: + for reset_state, _ in self._layers: reset_state() torch.cuda.synchronize(self._device) + # The kernels write the status and waited flags, and none runs now. + self.abort[_ABORT_STATUS:].zero_() + with self._abort_lock: + self._abort_words[_ABORT_REQUESTED] -= handled_requests def install_mask(self) -> None: """Write the active ranks of the EP group health into the device mask. Call only while no all-to-all collective is queued or running, and after :meth:`rearm` if a - collective aborted. The device is synchronized first, so no collective reads a partly - written mask. + collective aborted (see :meth:`rearm` for the order across ranks). The device is + synchronized first, so no collective reads a partly written mask. + + Raises: + RuntimeError: A bound layer dispatched without its combine, which would read the rows + of another mask, or the abort state is set. """ snapshot = self.ep_group_health.snapshot() if snapshot.generation == self._generation: return words = _mask_words(snapshot.mask, self.ep_rank) + if not all(between_rounds() for _, between_rounds in self._layers): + raise RuntimeError( + "Install the MoE All-to-All rank mask only between rounds: a bound layer " + "dispatched without its combine" + ) torch.cuda.synchronize(self._device) if self.aborted(): raise RuntimeError(f"MoE All-to-All abort state is {self.abort[_ABORT_WORDS].tolist()}") diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py index b7eaaa84c253..21092a45fe85 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py @@ -96,6 +96,8 @@ def _sentinel_env() -> tuple[str | None, ...]: def get_sync_protocol() -> str: """``TRTLLM_NVLINK_ONE_SIDED_A2A_SYNC_PROTOCOL``: "auto" (also when unset), "fence" or "sentinel". + Also warns once about each set variable that NVLinkOneSided no longer reads. + Raises: NVLinkOneSidedConfigError: If the value is none of these. """ @@ -929,12 +931,18 @@ def bind_execution_state(self, state: MoeExecutionState) -> None: ) if self.execution_state is not None: raise RuntimeError("NVLinkOneSided is already bound to an execution state") + # The mask holds one bit per rank of the state's EP group, which must be this layer's. + state_ep_size = state.ep_group_health.moe_world_size + if state_ep_size != self.ep_size: + raise ValueError( + f"An execution state of EP size {state_ep_size} cannot run EP size {self.ep_size}" + ) # The kernels treat the local rank as active, which the state checks for its own rank. if state.ep_rank != self.ep_rank: raise ValueError( f"An execution state of EP rank {state.ep_rank} cannot run EP rank {self.ep_rank}" ) - state.register_layer(self.reset_state) + state.register_layer(self.reset_state, self._mnnvl_checkpoint_is_idle) state.register_workspace( self.mnnvl_mem, self._reset_local_workspace, self._check_incomplete_round ) @@ -1436,8 +1444,10 @@ def reset_state(self) -> None: It resets this Python state machine only. After a sentinel dispatch without its combine, the native op still holds the round open, and the peers' rows of that round are still in flight; only re-initializing - the workspace (``moe_a2a_initialize``, collective) recovers it. The - in-tree caller (the warmup OOM handler) runs only without peers. + the workspace (``moe_a2a_initialize``, collective) recovers it. Of the + in-tree callers, the warmup OOM handler runs only without peers, and + ``MoeExecutionState.rearm`` also re-initializes this rank's row of + the workspace. """ self._dispatch_state = {"phase": "idle"} diff --git a/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py b/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py index c1d090fc51e8..a137c8c493f3 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py +++ b/tensorrt_llm/_torch/moe/fused_moe/ep_group_health.py @@ -17,11 +17,9 @@ This module provides :class:`EPGroupHealth`, a process-local, thread-safe data structure that records which Expert Parallel (EP) ranks the recovery coordinator -has committed as included vs. excluded from the data plane. It is consumed by: - - * AlltoAll communication backends (rank masking on dispatch / combine) - * The MoE load balancer (emergency-mask reconfiguration) - * The model engine and PyExecutor (degraded health reporting) +has committed as included vs. excluded from the data plane. Its consumer is +``MoeExecutionState``, which writes the active ranks into the device rank mask +that the NVLinkOneSided all-to-all kernels read on dispatch and combine. Detected or suspected physical liveness is separate evidence. Higher-layer coordination reconciles that evidence and commits membership; detectors and diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 2acaef637ad6..75ea5a9b6cbf 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -1404,8 +1404,6 @@ def _timed_load( validate_lm_head_output_dtype(model, self.llm_args.lm_head_output_dtype) if self.llm_args.enable_wide_ep_fault_tolerance: - # Imported here, so that a model without fault tolerance loads - # none of its modules. from .rank_failure_model_load import prepare_model prepare_model(model) diff --git a/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py index c61f04f67384..c4066021a85c 100644 --- a/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py +++ b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py @@ -19,15 +19,19 @@ from __future__ import annotations -from typing import List +from typing import TYPE_CHECKING import torch from ..moe.fused_moe.communication.execution_state import MoeExecutionState +from ..moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided from ..moe.fused_moe.ep_group_health import EPGroupHealth from ..moe.fused_moe.fused_moe_vanilla import VanillaMoE from ..moe.fused_moe.interface import MoE +if TYPE_CHECKING: + from tensorrt_llm.mapping import Mapping + # The alternative that each error about an unmet fault tolerance requirement offers. _TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" @@ -36,36 +40,51 @@ def prepare_model(model: torch.nn.Module) -> None: """Bind every MoE layer of ``model`` to one execution state over all EP ranks. Raises: - ValueError: A MoE layer's communication cannot be bound, or the model has no MoE layer. + ValueError: A MoE layer's communication cannot be bound, two MoE layers communicate over + different EP groups, or the model has no MoE layer. """ + # A model may run its MoE layers over an EP group other than its mapping's MoE parallelism; + # moe_comms checks that every layer runs over the same one. comms = moe_comms(model) - mapping = model.model_config.mapping state = MoeExecutionState( - EPGroupHealth(mapping.moe_ep_size), - mapping.moe_ep_rank, + EPGroupHealth(comms[0].ep_size), + comms[0].ep_rank, torch.device("cuda", torch.cuda.current_device()), ) for comm in comms: comm.bind_execution_state(state) -def moe_comms(model: torch.nn.Module) -> List[object]: +def moe_comms(model: torch.nn.Module) -> list[NVLinkOneSided]: """Return the communication strategies that ``model``'s MoE layers use, each once. Raises: - ValueError: A MoE layer's communication cannot be bound to an execution state. + ValueError: A MoE layer's communication cannot be bound to an execution state, or two + MoE layers communicate over different EP groups. """ comms = {} + first = None # The name and communication of the first MoE layer. for name, layer in model.named_modules(): if not isinstance(layer, (MoE, VanillaMoE)): continue comm = getattr(layer, "comm", None) - if not hasattr(comm, "bind_execution_state"): + if not isinstance(comm, NVLinkOneSided): uses = "no communication strategy" if comm is None else type(comm).__name__ raise ValueError( f"MoE layer {name} ({type(layer).__name__}) uses {uses}, but Wide-EP fault " "tolerance requires NVLinkOneSided, the only MoE communication that can leave " - f"out a failed rank. Select it or {_TURN_FAULT_TOLERANCE_OFF}." + f"out a failed rank. {_no_nvlink_one_sided_cause(layer.mapping)}; " + f"{_TURN_FAULT_TOLERANCE_OFF}." + ) + if first is None: + first = name, comm + first_name, first_comm = first + if (comm.ep_size, comm.ep_rank) != (first_comm.ep_size, first_comm.ep_rank): + raise ValueError( + f"MoE layer {name} communicates as EP rank {comm.ep_rank} of {comm.ep_size}, but " + f"MoE layer {first_name} as EP rank {first_comm.ep_rank} of {first_comm.ep_size}. " + "Wide-EP fault tolerance runs every MoE layer over one EP group; give the layers " + f"the same MoE parallelism or {_TURN_FAULT_TOLERANCE_OFF}." ) comms[id(comm)] = comm if not comms: @@ -73,3 +92,22 @@ def moe_comms(model: torch.nn.Module) -> List[object]: f"Wide-EP fault tolerance requires a model with MoE layers; {_TURN_FAULT_TOLERANCE_OFF}" ) return list(comms.values()) + + +def _no_nvlink_one_sided_cause(mapping: Mapping) -> str: + """Name the setting with which a MoE layer of ``mapping`` gets no NVLinkOneSided.""" + if not mapping.enable_attention_dp or mapping.dp_size == 1: + return ( + "A MoE layer gets it only under attention DP over more than one rank " + "(enable_attention_dp=True and tensor_parallel_size above 1)" + ) + if mapping.moe_tp_size != 1: + return ( + "A MoE layer gets it only without MoE TP (moe_tensor_parallel_size=1, not " + f"{mapping.moe_tp_size})" + ) + return ( + "The layer's MoE backend, or the communication method that TRTLLM_FORCE_COMM_METHOD or " + "the model forces, selected another; use a MoE backend that supports NVLinkOneSided and " + "leave TRTLLM_FORCE_COMM_METHOD unset or set it to NVLINK_ONE_SIDED" + ) diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 8bbcb9c8cb17..430e8e407e98 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -7306,6 +7306,42 @@ def reject(requirement: str, actual: str) -> NoReturn: reject(f"{name}=1", f"{name}={size}") +def _validate_wide_ep_fault_tolerance_launch(args: 'TorchLlmArgs') -> None: + """The launch rules of Wide-EP fault tolerance (its validator's first part). + + Every MoE layer must run the NVLink one-sided all-to-all of the built-in + model, the only MoE communication that can leave out a failed rank. A MoE + layer gets it only under attention DP over more than one rank, without MoE + TP and with expert parallelism over more than one rank. What these + arguments leave open (a MoE parallel split left unset, which a model may + resolve itself, and the MoE backend) is checked when the model loads. + """ + unmet = [] + if not args.enable_attention_dp: + unmet.append("attention DP (enable_attention_dp=True)") + if args.tensor_parallel_size == 1: + unmet.append("more than one attention DP rank (tensor_parallel_size " + "above 1, not 1)") + moe_tp_size = args.moe_tensor_parallel_size + if moe_tp_size is not None and moe_tp_size > 1: + unmet.append( + f"no MoE TP (moe_tensor_parallel_size=1, not {moe_tp_size})") + if args.moe_expert_parallel_size == 1: + unmet.append("expert parallelism over more than one rank " + "(moe_expert_parallel_size above 1, not 1)") + if args.dwdp_config is not None and args.dwdp_config.enable_experts: + unmet.append("no expert DWDP (dwdp_config.enable_experts=False), " + "which runs the MoE layers without expert parallelism") + if args.modeling_v2 == "require": + unmet.append("the built-in model (modeling_v2='auto' or 'off', not " + "'require'), because no modeling_v2 target runs its MoE " + "layers with fault tolerance") + if unmet: + raise ValueError("Wide-EP fault tolerance requires " + + "; ".join(unmet) + "; change the configuration or " + "set enable_wide_ep_fault_tolerance=False") + + class BaseLlmArgs(StrictBaseModel): """Base class for the LLM arguments. It contains all the common arguments.""" model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") @@ -8731,8 +8767,11 @@ def is_partial_model_loading(self) -> bool: default=False, description="Run the MoE all-to-all of wide expert parallelism in its " "fault-tolerant mode, in which the NVLink one-sided all-to-all " - "exchanges only with the EP ranks marked active. Requires the NVLink " - "one-sided all-to-all.", + "exchanges only with the EP ranks marked active. Requires attention " + "DP over more than one rank, no MoE TP and the built-in MoE layers. " + "Turns off the sentinel transport and the two-rank pull combine of " + "the NVLink one-sided all-to-all. Does not yet recover from a failed " + "rank.", status="prototype") reorder_policy_config: Optional[ReorderRequestPolicyConfig] = Field( @@ -9902,6 +9941,24 @@ def validate_load_balancer(self) -> 'TorchLlmArgs': self.moe_config.resolve_load_balancer_compatibility() return self + @model_validator(mode="after") + def validate_wide_ep_fault_tolerance(self) -> 'TorchLlmArgs': + """Refuse a fault-tolerant launch that these arguments rule out. + + ``_validate_wide_ep_fault_tolerance_launch`` checks the parallel layout + and model of the MoE all-to-all. ``modeling_v2='auto'`` falls back to + the built-in model, because no modeling_v2 target supports fault + tolerance yet. + """ + if self.enable_wide_ep_fault_tolerance: + _validate_wide_ep_fault_tolerance_launch(self) + if self.modeling_v2 == "auto": + logger.warning( + "modeling_v2: no target supports Wide-EP fault tolerance; " + "using the built-in implementation") + self.modeling_v2 = "off" + return self + @model_validator(mode="after") def validate_per_layer_dynamic_eplb_cuda_graph(self) -> 'TorchLlmArgs': """Per-layer dynamic EPLB plans cannot be captured in CUDA graphs.""" diff --git a/tensorrt_llm/schemas/trtllm-serve-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-config.schema.json index d40f08b33260..0b140a333a3c 100644 --- a/tensorrt_llm/schemas/trtllm-serve-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-config.schema.json @@ -8942,7 +8942,7 @@ }, "enable_wide_ep_fault_tolerance": { "default": false, - "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires attention DP over more than one rank, no MoE TP and the built-in MoE layers. Turns off the sentinel transport and the two-rank pull combine of the NVLink one-sided all-to-all. Does not yet recover from a failed rank.", "status": "prototype", "title": "Enable Wide Ep Fault Tolerance", "type": "boolean" diff --git a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json index b802362accbb..4456574d12d4 100644 --- a/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json +++ b/tensorrt_llm/schemas/trtllm-serve-disagg-config.schema.json @@ -2758,7 +2758,7 @@ }, "enable_wide_ep_fault_tolerance": { "default": false, - "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires attention DP over more than one rank, no MoE TP and the built-in MoE layers. Turns off the sentinel transport and the two-rank pull combine of the NVLink one-sided all-to-all. Does not yet recover from a failed rank.", "status": "prototype", "title": "Enable Wide Ep Fault Tolerance", "type": "boolean" @@ -11139,7 +11139,7 @@ }, "enable_wide_ep_fault_tolerance": { "default": false, - "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires the NVLink one-sided all-to-all.", + "description": "Run the MoE all-to-all of wide expert parallelism in its fault-tolerant mode, in which the NVLink one-sided all-to-all exchanges only with the EP ranks marked active. Requires attention DP over more than one rank, no MoE TP and the built-in MoE layers. Turns off the sentinel transport and the two-rank pull combine of the NVLink one-sided all-to-all. Does not yet recover from a failed rank.", "status": "prototype", "title": "Enable Wide Ep Fault Tolerance", "type": "boolean" diff --git a/tests/unittest/_torch/modeling/test_modeling_speculative.py b/tests/unittest/_torch/modeling/test_modeling_speculative.py index 35700a3b8d7f..ca2fe9f8631b 100644 --- a/tests/unittest/_torch/modeling/test_modeling_speculative.py +++ b/tests/unittest/_torch/modeling/test_modeling_speculative.py @@ -28,6 +28,7 @@ from tensorrt_llm._torch.attention.attention import Attention from tensorrt_llm._torch.attention.backends.interface import RopeParams from tensorrt_llm._torch.model_config import ModelConfig, MoESelection +from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import MistralConfigLoader from tensorrt_llm._torch.models.modeling_dflash import DFlashForCausalLM from tensorrt_llm._torch.models.modeling_speculative import ( Eagle3ForCausalLM, @@ -148,6 +149,68 @@ def build_draft( assert "preserve_kv_cache_quant_algo" not in target_config.extra_attrs +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "draft_kind", ["eagle3", "eagle3_mistral", "external_model", "external_drafter"] +) +@pytest.mark.parametrize("enabled", [False, True], ids=["off", "on"]) +def test_draft_config_carries_wide_ep_fault_tolerance( + monkeypatch: pytest.MonkeyPatch, draft_kind: str, enabled: bool +) -> None: + """A drafter's MoE layers bind to the target's execution state, so its config has the flag.""" + pretrained = PretrainedConfig( + hidden_size=128, + vocab_size=32, + num_hidden_layers=1, + torch_dtype=torch.bfloat16, + tie_word_embeddings=False, + architectures=["LlamaForCausalLM"], + ) + spec_kwargs = dict(max_draft_len=3, speculative_model="draft-checkpoint") + if draft_kind == "eagle3": + spec_config = Eagle3DecodingConfig(**spec_kwargs) + elif draft_kind == "eagle3_mistral": + spec_config = Eagle3DecodingConfig(**spec_kwargs, eagle3_model_arch="mistral_large3") + elif draft_kind == "external_model": + spec_config = MTPDecodingConfig(**spec_kwargs) + update_spec_config_from_model_config( + spec_config, + pretrained, + target_model_cls=SimpleNamespace(build_mtp_draft_model_from_config=True), + ) + else: + spec_config = DraftTargetDecodingConfig(**spec_kwargs) + target_config = ModelConfig( + pretrained_config=pretrained, + spec_config=spec_config, + enable_wide_ep_fault_tolerance=enabled, + skip_create_weights_in_init=True, + ) + draft_loads: list[dict] = [] + + def load_draft_config(*_args: object, **kwargs: object) -> ModelConfig: + draft_loads.append(kwargs) + return ModelConfig(pretrained_config=pretrained, skip_create_weights_in_init=True) + + monkeypatch.setattr( + ModelConfig, + "from_pretrained", + classmethod(lambda _cls, *args, **kwargs: load_draft_config(*args, **kwargs)), + ) + monkeypatch.setattr( + MistralConfigLoader, + "load", + lambda _loader, *args, **kwargs: load_draft_config(*args, **kwargs), + ) + monkeypatch.setattr("tensorrt_llm._torch.modules.linear.get_sm_version", lambda: 100) + monkeypatch.setattr(speculative_modeling, "get_spec_worker", lambda *args, **kwargs: None) + monkeypatch.setattr(speculative_modeling, "get_draft_model", lambda *args: nn.Module()) + with torch.device("cpu"): + SpecDecOneEngineForCausalLM(nn.Identity(), target_config) + + assert [kwargs["enable_wide_ep_fault_tolerance"] for kwargs in draft_loads] == [enabled] + + @pytest.mark.cpu_only def test_draft_kv_cache_quant_algo_override_updates_all_layers() -> None: target = SimpleNamespace( diff --git a/tests/unittest/_torch/moe/moe_a2a_test_utils.py b/tests/unittest/_torch/moe/moe_a2a_test_utils.py index 950f0bfc8c71..80769d8d8ce2 100644 --- a/tests/unittest/_torch/moe/moe_a2a_test_utils.py +++ b/tests/unittest/_torch/moe/moe_a2a_test_utils.py @@ -34,14 +34,30 @@ def a2a_workspace(ep_size: int, tokens: int = CAPACITY) -> tuple[torch.Tensor, . return workspace, meta, experts, payload.reshape(tokens, HIDDEN) -def a2a_dispatch(workspace, meta, experts, controls, payload, tokens=CAPACITY, rank=0): +def a2a_dispatch( + workspace: torch.Tensor, + meta: torch.Tensor, + experts: torch.Tensor, + controls: dict[str, bool | torch.Tensor], + payload: torch.Tensor, + tokens: int = CAPACITY, + rank: int = 0, +) -> tuple[list[torch.Tensor], int, torch.Tensor]: """Dispatch [payload, experts] like the MoE layer as ``rank``.""" ep_size = workspace.size(0) args = (experts, [payload, experts], workspace, meta, tokens, rank, ep_size, 1, ep_size, None) return torch.ops.trtllm.moe_a2a_dispatch(*args, **controls) -def a2a_combine(workspace, meta, payload, offset, controls, tokens=CAPACITY, rank=0): +def a2a_combine( + workspace: torch.Tensor, + meta: torch.Tensor, + payload: torch.Tensor, + offset: int, + controls: dict[str, bool | torch.Tensor], + tokens: int = CAPACITY, + rank: int = 0, +) -> torch.Tensor: """Combine like the MoE layer as ``rank``.""" args = (payload, tokens, workspace, meta, tokens, rank, workspace.size(0), 1, offset, False) return torch.ops.trtllm.moe_a2a_combine(*args, **controls) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py b/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py index 5b504277fddf..8a2612532ee5 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_nvlink_one_sided.py @@ -13,6 +13,7 @@ import pickle import sys +import time from collections.abc import Iterator from dataclasses import dataclass, replace from enum import Enum @@ -797,6 +798,25 @@ def test_nvlink_one_sided(case: Case, mpi_pools: dict[tuple[int, bool], MPIPoolE # CUDA graphs under an execution state # ============================================================================ +# Longer than the kernel timeouts the tests below set, so that a collective stuck on a peer fails on +# the waiting rank first. +_BARRIER_TIMEOUT_S = 120.0 +# Bounds a whole test, so that a worker stuck outside a barrier fails the test instead of the stage. +_WORKER_TIMEOUT_S = 600.0 + + +def _bounded_barrier() -> None: + """Wait for every rank; raise instead of hanging when a peer never arrives.""" + request = MPI.COMM_WORLD.Ibarrier() + deadline = time.monotonic() + _BARRIER_TIMEOUT_S + while not request.Test(): + if time.monotonic() > deadline: + raise TimeoutError( + f"rank {MPI.COMM_WORLD.Get_rank()} waited {_BARRIER_TIMEOUT_S:.0f} s at a barrier " + "that a peer never reached" + ) + time.sleep(0.01) + def _run_graph_replay_worker(failed_rank: int) -> dict: """Capture one round trip with every rank active, then replay it without ``failed_rank``.""" @@ -854,7 +874,7 @@ def round_trip() -> torch.Tensor: set_inputs(everyone, 0) check(torch.equal(round_trip(), x), "eager round combined wrong") torch.cuda.synchronize() - MPI.COMM_WORLD.Barrier() + _bounded_barrier() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): output = round_trip() @@ -862,7 +882,7 @@ def round_trip() -> torch.Tensor: graph.replay() torch.cuda.synchronize() check(torch.equal(output, x), "replay with every rank combined wrong") - MPI.COMM_WORLD.Barrier() + _bounded_barrier() # The failed rank replays nothing more; the others install the mask without it and # replay the graph captured before. @@ -876,7 +896,7 @@ def round_trip() -> torch.Tensor: graph.replay() torch.cuda.synchronize() check(torch.equal(output, x), f"replay {seed} without rank {failed_rank} wrong") - MPI.COMM_WORLD.Barrier() + _bounded_barrier() return {"rank": rank, "errors": errors, "sentinel_max_tokens": comm.sentinel_max_tokens} finally: comm.destroy() @@ -889,8 +909,14 @@ def test_graph_replay_follows_installed_mask() -> None: pytest.skip(f"requires {ep_size} GPUs") # A replay that waited for the masked rank would end at this kernel timeout. executor = MPIPoolExecutor(ep_size, env={"TRTLLM_MOE_COMM_TIMEOUT_SEC": "60"}) + finished = False try: - results = list(executor.map(_run_graph_replay_worker, [failed_rank] * ep_size)) + results = list( + executor.map( + _run_graph_replay_worker, [failed_rank] * ep_size, timeout=_WORKER_TIMEOUT_S + ) + ) + finished = True skipped = [r["skip"] for r in results if "skip" in r] if skipped: pytest.skip(skipped[0]) @@ -898,7 +924,8 @@ def test_graph_replay_follows_installed_mask() -> None: # The sentinel transport takes no active-rank mask, so every step takes the fence one. assert all(r["sentinel_max_tokens"] == 0 for r in results) finally: - executor.shutdown(wait=True, cancel_futures=True) + # Waiting for a worker that failed or timed out could hang the test. + executor.shutdown(wait=finished, cancel_futures=True) # ============================================================================ @@ -909,8 +936,6 @@ def test_graph_replay_follows_installed_mask() -> None: def _run_abort_stuck_collective_worker(failed_rank: int, stop_before: str, round_flag: int) -> dict: """Abort a round from ``round_flag`` whose ``failed_rank`` stops before ``stop_before``.""" # Import locally so cloudpickle does not serialize torch's dynamic ops namespace. - import time - import torch from tensorrt_llm._torch.moe.fused_moe.communication.execution_state import MoeExecutionState @@ -963,11 +988,11 @@ def round_trip(senders: list[int], seed: int) -> tuple[torch.Tensor, torch.Tenso try: state = MoeExecutionState(health, rank, torch.device("cuda")) comm.bind_execution_state(state) - MPI.COMM_WORLD.Barrier() + _bounded_barrier() x, combined = round_trip(everyone, 0) check(torch.equal(combined, x), "healthy round combined wrong") torch.cuda.synchronize() - MPI.COMM_WORLD.Barrier() + _bounded_barrier() # Round 1: the dispatch waits for flag value round_flag + 1 and the combine for # round_flag + 2, modulo 2**32. A wait publishes 1 for a flag value that wrapped to 0. @@ -1010,12 +1035,12 @@ def round_trip(senders: list[int], seed: int) -> tuple[torch.Tensor, torch.Tenso own = torch.arange(capacity, device=x.device) % ep_size == rank expected[own] = x[own] check(torch.equal(combined, expected), "the incomplete round combined wrong") - MPI.COMM_WORLD.Barrier() + _bounded_barrier() health.mark_failed(failed_rank) if rank in active: state.rearm() - MPI.COMM_WORLD.Barrier() + _bounded_barrier() if rank in active: state.install_mask() for seed in (2, 3): @@ -1023,7 +1048,7 @@ def round_trip(senders: list[int], seed: int) -> tuple[torch.Tensor, torch.Tenso check(torch.equal(combined, x), f"round {seed} after the recovery combined wrong") torch.cuda.synchronize() check(not state.aborted(), "a round after the recovery aborted") - MPI.COMM_WORLD.Barrier() + _bounded_barrier() return {"rank": rank, "errors": errors} finally: comm.destroy() @@ -1042,12 +1067,17 @@ def test_abort_releases_bound_collectives_waiting_on_a_stopped_rank( if torch.cuda.device_count() < ep_size: pytest.skip(f"requires {ep_size} GPUs") executor = MPIPoolExecutor(ep_size, env={"TRTLLM_MOE_COMM_TIMEOUT_SEC": "30"}) + finished = False try: args = [(failed_rank, stop_before, round_flag)] * ep_size - results = list(executor.starmap(_run_abort_stuck_collective_worker, args)) + results = list( + executor.starmap(_run_abort_stuck_collective_worker, args, timeout=_WORKER_TIMEOUT_S) + ) + finished = True skipped = [r["skip"] for r in results if "skip" in r] if skipped: pytest.skip(skipped[0]) assert [e for r in results for e in r["errors"]] == [] finally: - executor.shutdown(wait=True, cancel_futures=True) + # Waiting for a worker that failed or timed out could hang the test. + executor.shutdown(wait=finished, cancel_futures=True) diff --git a/tests/unittest/_torch/moe/test_communication_factory.py b/tests/unittest/_torch/moe/test_communication_factory.py index 4a18f85adcba..926091e4446b 100644 --- a/tests/unittest/_torch/moe/test_communication_factory.py +++ b/tests/unittest/_torch/moe/test_communication_factory.py @@ -613,22 +613,60 @@ def test_one_sided_runs_rank_mask_kernels_only_with_fault_tolerance( assert one_sided.call_args.kwargs["enable_wide_ep_fault_tolerance"] is fault_tolerance -@pytest.mark.parametrize("fault_tolerance", [False, True]) -def test_fault_tolerance_reports_one_sided_error( - monkeypatch: pytest.MonkeyPatch, fault_tolerance: bool -): +def test_fault_tolerance_reports_one_sided_error(monkeypatch: pytest.MonkeyPatch): """Fault tolerance runs only on NVLinkOneSided, so its construction error is not swallowed.""" model_config = _make_model_config(torch.float16) - model_config.enable_wide_ep_fault_tolerance = fault_tolerance + model_config.enable_wide_ep_fault_tolerance = True monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") - if fault_tolerance: - with pytest.raises(ValueError, match="requires the NVLinkOneSided.*strategy unavailable"): - _create_one_sided(model_config, forced=False) + with pytest.raises(ValueError, match="requires the NVLinkOneSided.*strategy unavailable"): + _create_one_sided(model_config, forced=False) + + +@pytest.mark.parametrize("fault_tolerance", [False, True]) +@pytest.mark.parametrize( + ("name", "value", "warned"), + [ + ("TRTLLM_ALLTOALL_WATCHDOG_TIMEOUT_S", "5", "the all-to-all watchdog was removed"), + ("TRTLLM_ALLTOALL_WATCHDOG_POLL_INTERVAL_S", "1", "the all-to-all watchdog was removed"), + ("TLLM_FAULT_TOLERANCE_MODE", "1", "set enable_wide_ep_fault_tolerance=True"), + ("TLLM_FAULT_TOLERANCE_MODE", "0", None), + ], + ids=["watchdog-timeout", "watchdog-poll-interval", "mode-1", "mode-0"], +) +def test_removed_fault_tolerance_setting_warns( + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, + warned: str | None, + fault_tolerance: bool, +): + """Every strategy selection warns about a set variable of the earlier fault tolerance mode.""" + for removed in ( + *communication_factory._REMOVED_WATCHDOG_ENVS, + communication_factory._FAULT_TOLERANCE_MODE_ENV, + ): + monkeypatch.delenv(removed, raising=False) + monkeypatch.setenv(name, value) + warnings = [] + monkeypatch.setattr( + communication_factory.logger, + "warning_once", + lambda message, key: warnings.append((key, message)), + ) + # Without attention DP the factory selects no all-to-all at all. + model_config = _make_model_config() + model_config.mapping.enable_attention_dp = False + model_config.enable_wide_ep_fault_tolerance = fault_tolerance + + assert _create_one_sided(model_config, forced=False) is None + if warned is None: + assert warnings == [] else: - assert isinstance(_create_one_sided(model_config, forced=False), AllGatherReduceScatter) + assert [key for key, _ in warnings] == [f"removed_env_{name}"] + assert warned in warnings[0][1] _NCCL_EP_QUANT_ALGOS = [ diff --git a/tests/unittest/_torch/moe/test_moe_a2a_abort.py b/tests/unittest/_torch/moe/test_moe_a2a_abort.py index 9e5cd14bce20..67054ca4b659 100644 --- a/tests/unittest/_torch/moe/test_moe_a2a_abort.py +++ b/tests/unittest/_torch/moe/test_moe_a2a_abort.py @@ -34,6 +34,7 @@ # The abort state holds ``requested`` and ``status``, then the flag value awaited from each peer. _ABORT_WORDS = int(internal.thop.MOE_A2A_ABORT_STATE_WORDS) +_WAITED_FLAG = int(internal.thop.MOE_A2A_ABORT_WAITED_FLAG_INDEX) def _offset(meta: torch.Tensor, name: str) -> int: @@ -147,16 +148,18 @@ def test_no_peer_writes_after_incomplete_round(ep_size: int, rank: int) -> None: assert torch.equal(torch.cat([workspace[:rank], workspace[rank + 1 :]]), peers) +@pytest.mark.parametrize("use_low_precision", [False, True], ids=["bf16_combine", "fp8_combine"]) @pytest.mark.parametrize( "ep_size, rank, peer", [(2, 0, 1), (72, 34, 2)], ids=["two_ranks", "rank_34_of_72"] ) def test_aborted_dispatch_uses_no_rows_of_unconfirmed_peers( - ep_size: int, rank: int, peer: int + ep_size: int, rank: int, peer: int, use_low_precision: bool ) -> None: """An aborted round uses no rows of a peer whose completion flag it did not see. The peer writes its receive count but never its flag, and again after the abort. Rank 34 of 72 - waits on peer 2 and then on itself in the same lane, and keeps its own rows. + waits on peer 2 and then on itself in the same lane, and keeps its own rows. With + ``use_low_precision``, the combine stages and exchanges the rows in FP8. """ workspace, meta, experts, payload = a2a_workspace(ep_size) torch.ops.trtllm.moe_a2a_initialize(workspace, meta, rank, ep_size, barrier=False) @@ -179,15 +182,23 @@ def test_aborted_dispatch_uses_no_rows_of_unconfirmed_peers( assert torch.all(expert_ids[peer] == -1) assert torch.equal(expert_ids[rank], experts) - # The rows of the peer in this rank's combine input region, which the peer reads. - stride = HIDDEN * payload.element_size() + # The rows of the peer in this rank's combine input region, which the peer reads. A low-precision + # combine stages a payload outside the workspace there at one byte per element. + stride = HIDDEN * (1 if use_low_precision else payload.element_size()) first = offset + peer * CAPACITY * stride peer_rows = workspace[rank, first : first + CAPACITY * stride] peer_rows.fill_(0x7F) - output = a2a_combine(workspace, meta, received[0], offset, controls, rank=rank) + combine_controls = {**controls, "use_low_precision": use_low_precision} + output = a2a_combine(workspace, meta, received[0], offset, combine_controls, rank=rank) _finish(abort, "the combine of the incomplete round waited for the peer") assert torch.all(peer_rows == 0x7F) - torch.testing.assert_close(output, payload, rtol=0, atol=0) + expected = payload + if use_low_precision: + # The FP8 conversion rounds to nearest and saturates, and every FP8 value is exact in bf16. + fp8_max = torch.finfo(torch.float8_e4m3fn).max + expected = payload.float().clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn) + expected = expected.to(payload.dtype) + torch.testing.assert_close(output, expected, rtol=0, atol=0) def _check_incomplete_round(workspace, meta, abort, rank=0) -> None: @@ -274,15 +285,6 @@ def test_own_round_value_keeps_peer_rows() -> None: assert peer_abort[:2].tolist() == [0, 1] -def test_abort_state_must_be_pinned() -> None: - """The kernels read the abort state at its host address, so the ops refuse pageable memory.""" - workspace, meta, experts, payload = a2a_workspace(2) - controls, _ = _rank_mask_controls(3) - controls["abort_state"] = torch.zeros(_ABORT_WORDS, dtype=torch.int32) - with pytest.raises(RuntimeError, match="pinned"): - a2a_dispatch(workspace, meta, experts, controls, payload) - - def test_incomplete_dispatch_marks_active_peers() -> None: """A dispatch whose round is incomplete marks the rounds of the active peers incomplete.""" workspace, meta, experts, payload = a2a_workspace(3) @@ -295,37 +297,92 @@ def test_incomplete_dispatch_marks_active_peers() -> None: assert [_word(workspace, incomplete_round, row=r).item() for r in range(3)] == [5, 5, 0] -def _fence_timeout() -> None: +def _set_flag(word: torch.Tensor, value: int) -> None: + """Store the uint32 round flag ``value`` in an int32 workspace word.""" + word.fill_(value - 2**32 if value >= 2**31 else value) + + +@pytest.mark.parametrize("flag_val", [0, 2**32 - 2], ids=["first_round", "flag_wraps"]) +def test_aborted_combine_reduces_own_rows(flag_val: int) -> None: + """A combine that an abort request releases reduces only this rank's own rows. + + Peer 1 completes the dispatch but never signals the combine. While the combine waits, it + publishes the flag value it waits for. The abort request ends the wait, withdraws that value and + marks the round incomplete. With ``flag_wraps``, the combine flag value wraps to 0. + """ + workspace, meta, experts, payload = a2a_workspace(2) + controls, abort = _rank_mask_controls(3) + _set_flag(_word(workspace, _offset(meta, "FLAG_VAL")), flag_val) + dispatch_flag, combine_flag = (flag_val + 1) % 2**32, (flag_val + 2) % 2**32 + # Peer 1's dispatch flag for rank 0, and the combine flag value of peer 1's previous round. + _set_flag(_word(workspace, _offset(meta, "DISPATCH_COMPLETION_FLAGS") + 4), dispatch_flag) + _set_flag(_word(workspace, _offset(meta, "COMBINE_COMPLETION_FLAGS") + 4), flag_val) + experts.copy_(torch.arange(CAPACITY, dtype=torch.int32).view(-1, 1) % 2) + received, offset, _ = a2a_dispatch(workspace, meta, experts, controls, payload) + _finish(abort, "the dispatch waited for peer 1 after its flag was set") + # Rank 0's rows in peer 1's combine input region, which peer 1 never prepared. + rows_bytes = CAPACITY * HIDDEN * payload.element_size() + workspace[1, offset : offset + rows_bytes].fill_(0x3F) + + output = a2a_combine(workspace, meta, received[0], offset, controls) + waited = abort[_WAITED_FLAG + 1] + deadline = time.monotonic() + 10 + while waited.item() == 0 and time.monotonic() < deadline: + time.sleep(0.01) + published = waited.item() + abort[0] = 1 + _finish(abort, "the abort request did not release the combine's wait") + # A wrapped combine flag value of 0 publishes 1, as 0 means no wait. + assert published == max(combine_flag, 1) + assert waited.item() == 0 + assert abort[:2].tolist() == [1, 1] + assert _word(workspace, _offset(meta, "FLAG_VAL")).item() % 2**32 == combine_flag + assert _word(workspace, _offset(meta, "INCOMPLETE_ROUND")).item() == combine_flag | 1 + expected = torch.where(experts == 0, payload, torch.zeros_like(payload)) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + +def _fence_timeout(with_abort_state: bool) -> None: """Check that a fence wait on a silent peer still fails at the timeout.""" torch.ops.trtllm.moe_a2a_set_timeout(1) workspace, meta, experts, payload = a2a_workspace(2) - controls, _ = _rank_mask_controls(3) + controls, abort = _rank_mask_controls(3) + if not with_abort_state: + del controls["abort_state"] a2a_dispatch(workspace, meta, experts, controls, payload) try: torch.cuda.synchronize() except RuntimeError: # Exit at once, flushing the device printf C stdio holds on a pipe. print("fence timeout trapped", flush=True) + if with_abort_state: + waited = abort[_WAITED_FLAG + 1].item() + print(f"abort state {abort[:2].tolist()} waited flag {waited}", flush=True) ctypes.CDLL(None).fflush(None) os._exit(0) raise AssertionError("dispatch completed without its peer") -def test_fence_timeout_still_traps() -> None: +@pytest.mark.parametrize("with_abort_state", [False, True], ids=["no_abort_state", "abort_state"]) +def test_fence_timeout_still_traps(with_abort_state: bool) -> None: # The child traps on purpose, in a fresh interpreter without the launcher's environment. launcher_prefixes = ("SLURM_", "PMIX_", "PMI_", "OMPI_") env = {k: v for k, v in os.environ.items() if not k.startswith(launcher_prefixes)} # The child imports the shared helpers of the unit tests, as pytest makes them importable. unittest_root = str(Path(__file__).resolve().parents[2]) env["PYTHONPATH"] = os.pathsep.join(p for p in (unittest_root, env.get("PYTHONPATH")) if p) + mode = "abort_state" if with_abort_state else "no_abort_state" result = subprocess.run( - [sys.executable, __file__], env=env, capture_output=True, text=True, timeout=300 + [sys.executable, __file__, mode], env=env, capture_output=True, text=True, timeout=300 ) assert result.returncode == 0, result.stdout + result.stderr # The kernel's own report, not just any device error. assert "dispatch: rank 0 timed out waiting for rank 1 flag=" in result.stdout, result.stdout assert "fence timeout trapped" in result.stdout, result.stdout + if with_abort_state: + # Nothing requested an abort, and the trap left the published flag value of the first dispatch. + assert "abort state [0, 0] waited flag 1" in result.stdout, result.stdout if __name__ == "__main__": - _fence_timeout() + _fence_timeout(with_abort_state=sys.argv[1:] == ["abort_state"]) diff --git a/tests/unittest/_torch/moe/test_moe_execution_state.py b/tests/unittest/_torch/moe/test_moe_execution_state.py index 2d5f01000c89..1acd59e81afc 100644 --- a/tests/unittest/_torch/moe/test_moe_execution_state.py +++ b/tests/unittest/_torch/moe/test_moe_execution_state.py @@ -26,7 +26,7 @@ import tensorrt_llm # noqa: F401 from tensorrt_llm._torch.moe.fused_moe import ConfigurableMoE, TritonFusedMoE, VanillaMoE -from tensorrt_llm._torch.moe.fused_moe.communication import AllGatherReduceScatter +from tensorrt_llm._torch.moe.fused_moe.communication import AllGatherReduceScatter, execution_state from tensorrt_llm._torch.moe.fused_moe.communication.execution_state import MoeExecutionState from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided from tensorrt_llm._torch.moe.fused_moe.ep_group_health import EPGroupHealth @@ -43,6 +43,7 @@ def _bindable_state(**attributes) -> SimpleNamespace: """Return a stand-in for an execution state that records the layers bound to it.""" + attributes.setdefault("ep_group_health", EPGroupHealth(_EP_SIZE)) return SimpleNamespace(register_layer=Mock(), register_workspace=Mock(), **attributes) @@ -126,12 +127,15 @@ def reset_workspace() -> None: _collective_round(state, workspace, meta, experts, payload) done = _queued_work_done() - # A long wait names the flag value it expects from rank 1. It is not an abort. - _wait_for(state.waited_flags, state, "the dispatch published no waited flag") - assert state.waited_flags() == {1: 1} - assert not state.aborted() - requester = threading.Thread(target=state.request_abort) - requester.start() + try: + # A long wait names the flag value it expects from rank 1. It is not an abort. + _wait_for(state.waited_flags, state, "the dispatch published no waited flag") + assert state.waited_flags() == {1: 1} + assert not state.aborted() + finally: + # Also on a failed check, so that the dispatch does not keep waiting for rank 1. + requester = threading.Thread(target=state.request_abort) + requester.start() requester.join(_JOIN_TIMEOUT_S) assert not requester.is_alive(), "abort request queued behind the spinning kernel" _wait_for(done, state, "collective kept waiting after the abort request") @@ -211,23 +215,63 @@ def test_rearm_resets_bound_layers_and_shared_workspaces() -> None: assert comm._dispatch_state == {"phase": "idle"} -def test_graph_replay_follows_installed_mask() -> None: - """A CUDA graph captured with every rank active replays without the rank masked out later.""" - workspace, meta, experts, payload = a2a_workspace(_EP_SIZE) - health = EPGroupHealth(_EP_SIZE) - state = MoeExecutionState(health, 0, workspace.device) - torch.cuda.synchronize() - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - output = _collective_round(state, workspace, meta, experts, payload) +def test_abort_requested_during_rearm_stays(monkeypatch: pytest.MonkeyPatch) -> None: + """Rearm clears the requests made before it, the status and the waited flags, but not a + request made while it waits for the device.""" + health = EPGroupHealth(2) + state = MoeExecutionState(health, 0, torch.device("cuda")) + state.request_abort() + state.abort[1] = 1 # a kernel stopped waiting + state.abort[_WAITED_FLAGS + 1] = 7 + synchronize = torch.cuda.synchronize + requests = [state.request_abort] + + def synchronize_then_request(device=None) -> None: + synchronize(device) + while requests: + requests.pop()() + + monkeypatch.setattr(torch.cuda, "synchronize", synchronize_then_request) + state.rearm() + assert state.abort[:2].tolist() == [1, 0] + assert not state.abort[_WAITED_FLAGS:].any() + health.mark_failed(1) + with pytest.raises(RuntimeError, match="abort state"): + state.install_mask() + state.rearm() + assert not state.aborted() + state.install_mask() + assert state.active_rank_mask.cpu().tolist() == [1, 0, 0, 0] + +def test_install_mask_refused_mid_round() -> None: + """A layer between its dispatch and its combine keeps the mask of its dispatch.""" + health = EPGroupHealth(2) + state = MoeExecutionState(health, 0, torch.device("cuda")) + comm = _frontend() + comm.bind_execution_state(state) + comm._dispatch_state = {"phase": "dispatched"} health.mark_failed(1) + with pytest.raises(RuntimeError, match="only between rounds"): + state.install_mask() + assert state.active_rank_mask.cpu().tolist() == [3, 0, 0, 0] + comm.reset_state() state.install_mask() - for _ in range(2): - graph.replay() - _wait_for(_queued_work_done(), state, "the replay waited for the masked-out rank") - torch.testing.assert_close(output, payload, rtol=0, atol=0) - assert not state.aborted() + assert state.active_rank_mask.cpu().tolist() == [1, 0, 0, 0] + + +@pytest.mark.cpu_only +def test_state_refused_under_confidential_computing(monkeypatch: pytest.MonkeyPatch) -> None: + """The kernels read the abort state in pinned host memory, which Confidential Computing rules out.""" + monkeypatch.setattr(execution_state, "confidential_compute_enabled", lambda: True) + with pytest.raises(RuntimeError, match="not supported with Confidential Computing"): + MoeExecutionState(EPGroupHealth(2), 0, torch.device("cuda")) + + +def test_abort_state_is_pinned() -> None: + """The kernels read the abort state in place from host memory, so it is pinned when built.""" + state = MoeExecutionState(EPGroupHealth(2), 0, torch.device("cuda")) + assert state.abort.is_pinned() def _frontend(*, fault_tolerance: bool = True, ep_rank: int = 0) -> NVLinkOneSided: @@ -259,6 +303,9 @@ def test_bind_refuses_unsupported_layers() -> None: _frontend(fault_tolerance=False).bind_execution_state(state) with pytest.raises(ValueError, match="cannot run EP rank 1"): _frontend(ep_rank=1).bind_execution_state(state) + other_group = _bindable_state(ep_rank=0, ep_group_health=EPGroupHealth(2 * _EP_SIZE)) + with pytest.raises(ValueError, match=f"EP size {2 * _EP_SIZE} cannot run EP size {_EP_SIZE}"): + _frontend().bind_execution_state(other_group) @pytest.mark.cpu_only @@ -306,7 +353,7 @@ def combine_op(*args, **kwargs): assert kwargs["enable_rank_mask"] is True and kwargs["use_sentinel"] is False assert kwargs["active_rank_mask"] is state.active_rank_mask assert kwargs["abort_state"] is state.abort - state.register_layer.assert_called_once_with(bound.reset_state) + state.register_layer.assert_called_once_with(bound.reset_state, bound._mnnvl_checkpoint_is_idle) state.register_workspace.assert_called_once_with( bound.mnnvl_mem, bound._reset_local_workspace, bound._check_incomplete_round ) @@ -321,27 +368,97 @@ def _moe_layer(cls: type, **attributes) -> torch.nn.Module: return layer +def _layer_mapping(attention_dp: bool = True, dp_size: int = 2, moe_tp_size: int = 1): + """Return the mapping fields that tell why a MoE layer has no NVLinkOneSided.""" + return SimpleNamespace( + enable_attention_dp=attention_dp, dp_size=dp_size, moe_tp_size=moe_tp_size + ) + + +_BACKEND_CAUSE = "The layer's MoE backend, or the communication method that TRTLLM_FORCE_COMM" +_ATTENTION_DP_CAUSE = "only under attention DP over more than one rank" + + @pytest.mark.cpu_only @pytest.mark.parametrize( - "kind, uses", + "kind, mapping, uses, cause", [ - ("all_gather", "(ConfigurableMoE) uses AllGatherReduceScatter"), - ("no_communication", "(ConfigurableMoE) uses no communication strategy"), - ("triton", "(TritonFusedMoE) uses no communication strategy"), - ("vanilla", "(VanillaMoE) uses no communication strategy"), + ( + "all_gather", + _layer_mapping(), + "(ConfigurableMoE) uses AllGatherReduceScatter", + _BACKEND_CAUSE, + ), + ( + "all_gather", + _layer_mapping(moe_tp_size=2), + "(ConfigurableMoE) uses AllGatherReduceScatter", + "only without MoE TP (moe_tensor_parallel_size=1, not 2)", + ), + ( + "no_communication", + _layer_mapping(), + "(ConfigurableMoE) uses no communication strategy", + _BACKEND_CAUSE, + ), + ( + "no_communication", + _layer_mapping(attention_dp=False, dp_size=1), + "(ConfigurableMoE) uses no communication strategy", + _ATTENTION_DP_CAUSE, + ), + ( + "no_communication", + _layer_mapping(dp_size=1), + "(ConfigurableMoE) uses no communication strategy", + _ATTENTION_DP_CAUSE, + ), + ( + "triton", + _layer_mapping(), + "(TritonFusedMoE) uses no communication strategy", + _BACKEND_CAUSE, + ), + ( + "vanilla", + _layer_mapping(), + "(VanillaMoE) uses no communication strategy", + _BACKEND_CAUSE, + ), ], ) -def test_unbindable_moe_layer_is_refused(kind: str, uses: str) -> None: - """Every MoE layer must bind, whatever its class.""" +def test_unbindable_moe_layer_is_refused(kind: str, mapping, uses: str, cause: str) -> None: + """Every MoE layer must bind, whatever its class; the refusal names the cause.""" layer = { - "all_gather": lambda: _moe_layer(ConfigurableMoE, comm=AllGatherReduceScatter(Mapping())), - "no_communication": lambda: _moe_layer(ConfigurableMoE, comm=None), - "triton": lambda: _moe_layer(TritonFusedMoE), - "vanilla": lambda: _moe_layer(VanillaMoE), + "all_gather": lambda: _moe_layer( + ConfigurableMoE, comm=AllGatherReduceScatter(Mapping()), mapping=mapping + ), + "no_communication": lambda: _moe_layer(ConfigurableMoE, comm=None, mapping=mapping), + "triton": lambda: _moe_layer(TritonFusedMoE, mapping=mapping), + "vanilla": lambda: _moe_layer(VanillaMoE, mapping=mapping), }[kind]() bound = _moe_layer(ConfigurableMoE, comm=_frontend()) with pytest.raises(ValueError, match=re.escape(f"MoE layer 1 {uses}")) as error: moe_comms(torch.nn.ModuleList([bound, layer])) + assert cause in str(error.value) + assert "enable_wide_ep_fault_tolerance=False" in str(error.value) + + +@pytest.mark.cpu_only +def test_layers_over_different_ep_groups_refused() -> None: + """Every MoE layer must run over the same EP group as the first one.""" + layers = [ + _moe_layer(ConfigurableMoE, comm=_frontend(ep_rank=1)), + _moe_layer(ConfigurableMoE, comm=_frontend(ep_rank=0)), + ] + with pytest.raises( + ValueError, + match=re.escape( + f"MoE layer 1 communicates as EP rank 0 of {_EP_SIZE}, but MoE layer 0 as EP rank 1 " + f"of {_EP_SIZE}" + ), + ) as error: + moe_comms(torch.nn.ModuleList(layers)) assert "enable_wide_ep_fault_tolerance=False" in str(error.value) @@ -353,18 +470,64 @@ def test_model_without_moe_layers_refused() -> None: @pytest.mark.cpu_only def test_layers_bound_to_one_execution_state(monkeypatch: pytest.MonkeyPatch) -> None: - """prepare_model binds every MoE layer to one execution state over every EP rank.""" + """prepare_model binds every MoE layer to one execution state over the layers' EP group. + + The model mapping's MoE parallelism (no expert parallelism here) differs from the layers' EP + group, as for a model that builds its own mapping for its routed experts. + """ monkeypatch.setattr( rank_failure_model_load, "MoeExecutionState", lambda health, ep_rank, device: SimpleNamespace(ep_group_health=health, ep_rank=ep_rank), ) monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) - comms = [Mock(spec=["bind_execution_state"]) for _ in range(2)] + comms = [Mock(spec=NVLinkOneSided, ep_size=4, ep_rank=1) for _ in range(2)] model = torch.nn.ModuleList([_moe_layer(ConfigurableMoE, comm=comm) for comm in comms]) - model.model_config = SimpleNamespace(mapping=SimpleNamespace(moe_ep_size=4, moe_ep_rank=1)) + model.model_config = SimpleNamespace(mapping=SimpleNamespace(moe_ep_size=1, moe_ep_rank=0)) prepare_model(model) (state,), (other,) = (comm.bind_execution_state.call_args.args for comm in comms) assert state is other assert state.ep_rank == 1 assert state.ep_group_health.get_mask() == 0b1111 + + +def _target_with_drafter( + target_comm: NVLinkOneSided, draft_comm: NVLinkOneSided +) -> torch.nn.Module: + """Build a model with one MoE layer and a one-model drafter (``draft_model``) with another.""" + model = torch.nn.Module() + model.moe = _moe_layer(ConfigurableMoE, comm=target_comm) + model.draft_model = torch.nn.Module() + model.draft_model.moe = _moe_layer(ConfigurableMoE, comm=draft_comm) + return model + + +@pytest.mark.cpu_only +def test_drafter_layers_bound_to_the_target_state(monkeypatch: pytest.MonkeyPatch) -> None: + """prepare_model binds the drafter's MoE layers to the execution state of the target's.""" + monkeypatch.setattr( + rank_failure_model_load, + "MoeExecutionState", + lambda health, ep_rank, device: _bindable_state(ep_group_health=health, ep_rank=ep_rank), + ) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + target, drafter = _frontend(), _frontend() + prepare_model(_target_with_drafter(target, drafter)) + assert target.execution_state is not None + assert drafter.execution_state is target.execution_state + assert target.execution_state.register_layer.call_count == 2 + + +@pytest.mark.cpu_only +def test_drafter_over_another_ep_group_refused() -> None: + """A drafter whose MoE layers run as another EP rank than the target's is refused.""" + model = _target_with_drafter(_frontend(ep_rank=0), _frontend(ep_rank=1)) + with pytest.raises( + ValueError, + match=re.escape( + f"MoE layer draft_model.moe communicates as EP rank 1 of {_EP_SIZE}, but MoE layer " + f"moe as EP rank 0 of {_EP_SIZE}" + ), + ) as error: + prepare_model(model) + assert "enable_wide_ep_fault_tolerance=False" in str(error.value) diff --git a/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py index a16ba19f8973..095621d25bbe 100644 --- a/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py +++ b/tests/unittest/_torch/pyexecutor/test_model_loader_gms.py @@ -20,6 +20,7 @@ from tensorrt_llm._torch.moe.fused_moe.configurable_moe import ConfigurableMoE from tensorrt_llm._torch.moe.fused_moe.cutlass import TrtllmCutlassUnquantizedImpl from tensorrt_llm._torch.pyexecutor import model_loader as model_loader_mod +from tensorrt_llm._torch.pyexecutor import rank_failure_model_load from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader from tensorrt_llm._torch.weight_sharing import ( ARTIFACT_IDENTITY_FORMAT_VERSION, @@ -356,6 +357,32 @@ def test_gms_load_branch(monkeypatch, is_rw, expected_events): assert loader.weight_mapper is None +def test_wide_ep_fault_tolerance_binds_moe_layers_before_weight_load(monkeypatch): + """Under Wide-EP fault tolerance, the loader binds the MoE layers before it loads weights.""" + events = [] + loader = _make_loader(monkeypatch, events=events) + loader.llm_args.enable_wide_ep_fault_tolerance = True + monkeypatch.setattr( + rank_failure_model_load, "prepare_model", lambda model: events.append("prepare_model") + ) + _install_gms_backend(monkeypatch, _build_gms_backend(is_rw=True, events=events)) + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "HF" + checkpoint_loader.load_weights.return_value = {"weight": MagicMock()} + + loader.load("/ckpt", checkpoint_loader) + + assert events == [ + "prepare_model", + "pool_enter", + "_apply", + "to", + "load_weights", + "post_load_weights", + "pool_exit", + ] + + @pytest.mark.parametrize( ("is_rw", "include_draft"), [(True, False), (True, True), (False, False)], diff --git a/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py b/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py index cc324e95bbb5..d2ad191d6387 100644 --- a/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py +++ b/tests/unittest/_torch/test_mnnvl_alltoall_workspace.py @@ -157,7 +157,8 @@ def test_checkpoint_prepare_timeout_fails_closed(monkeypatch) -> None: def test_checkpoint_prepare_rejects_remote_active_client() -> None: lifecycle, memory, _ = _make_lifecycle() memory.comm = _FakeComm(clients_idle_by_rank=[True, False]) - lifecycle.register(_Client()) + client = _Client() + lifecycle.register(client) with pytest.raises(RuntimeError, match=r"active MoE All-to-All phase on ranks \[1\]"): lifecycle.checkpoint_prepare() diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 90cfffa465d0..43b88650243e 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -155,12 +155,99 @@ def test_checkpoint_io_policy_rejects_unknown_value() -> None: TorchLlmArgs(model=llama_model_path, checkpoint_io_policy="unknown") +# The smallest configuration that Wide-EP fault tolerance accepts. +_FAULT_TOLERANT_ARGS = dict(enable_wide_ep_fault_tolerance=True, + enable_attention_dp=True, + tensor_parallel_size=2) + + +@pytest.mark.cpu_only +def test_wide_ep_fault_tolerance_launch_checked_on_build() -> None: + with pytest.raises( + ValidationError, + match=re.escape("attention DP (enable_attention_dp=True)")): + TorchLlmArgs(model=llama_model_path, + **{ + **_FAULT_TOLERANT_ARGS, "enable_attention_dp": False + }) + + @pytest.mark.cpu_only -def test_wide_ep_fault_tolerance_defaults_off( - monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("TLLM_FAULT_TOLERANCE_MODE", raising=False) - assert not TorchLlmArgs( - model=llama_model_path).enable_wide_ep_fault_tolerance +@pytest.mark.parametrize(("fields", "fix"), [ + ({ + "enable_attention_dp": False + }, "attention DP (enable_attention_dp=True)"), + ({ + "tensor_parallel_size": 1 + }, "more than one attention DP rank (tensor_parallel_size above 1, not 1)"), + ({ + "moe_tensor_parallel_size": 2 + }, "no MoE TP (moe_tensor_parallel_size=1, not 2)"), + ({ + "moe_expert_parallel_size": 1 + }, "expert parallelism over more than one rank " + "(moe_expert_parallel_size above 1, not 1)"), + ({ + "dwdp_config": DwdpConfig(dwdp_size=2) + }, "no expert DWDP (dwdp_config.enable_experts=False)"), + ({ + "modeling_v2": "require" + }, "the built-in model (modeling_v2='auto' or 'off', not 'require')"), +]) +def test_wide_ep_fault_tolerance_launch_rejected(fields: dict[str, Any], + fix: str) -> None: + """The arguments rule out MoE layers without NVLink one-sided.""" + args = TorchLlmArgs(model=llama_model_path, **_FAULT_TOLERANT_ARGS) + with pytest.raises(ValueError, match=re.escape(fix)) as error: + args.model_copy(update=fields).validate_wide_ep_fault_tolerance() + assert "set enable_wide_ep_fault_tolerance=False" in str(error.value) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("fields", [{}, { + "modeling_v2": "auto" +}], + ids=["default", "explicit"]) +def test_wide_ep_fault_tolerance_uses_builtin_model( + fields: dict[str, Any]) -> None: + """Fault tolerance turns modeling_v2 'auto' off with one warning.""" + with patch.object(llm_args_mod.logger, "warning") as warning: + args = TorchLlmArgs(model=llama_model_path, + **_FAULT_TOLERANT_ARGS, + **fields) + assert args.modeling_v2 == "off" + messages = [call.args[0] for call in warning.call_args_list] + assert sum("using the built-in implementation" in message + for message in messages) == 1 + assert TorchLlmArgs(model=llama_model_path, **fields).modeling_v2 == "auto" + + +@pytest.mark.cpu_only +def test_wide_ep_fault_tolerance_reaches_the_model_config() -> None: + """The loader builds the model's config with the field.""" + + class _StopAfterLoadConfig(Exception): + pass + + args = TorchLlmArgs(model="/tmp/dummy_model", **_FAULT_TOLERANT_ARGS) + checkpoint_loader = create_autospec(HfCheckpointLoader, + instance=True, + spec_set=True) + checkpoint_loader.load_config.side_effect = _StopAfterLoadConfig + loader = model_loader_mod.ModelLoader( + args, + args.parallel_config.to_mapping(), + spec_config=None, + sparse_attention_config=None, + max_num_tokens=args.max_num_tokens, + max_seq_len=args.max_seq_len, + ) + + with pytest.raises(_StopAfterLoadConfig): + loader._load_and_validate_config("/tmp/dummy_model", checkpoint_loader) + + kwargs = checkpoint_loader.load_config.call_args.kwargs + assert kwargs["enable_wide_ep_fault_tolerance"] is True @pytest.mark.cpu_only From 20cc6c3d54770fc914338917874d7f8279f6438a Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Thu, 19 Nov 2026 14:47:49 -0800 Subject: [PATCH 04/13] [None][test] Set enable_wide_ep_fault_tolerance in the DSpark EPLB tests' model config The drafter's model config now forwards enable_wide_ep_fault_tolerance from the target's, so the tests' model config stub carries the field, as a real ModelConfig does. Signed-off-by: Andrew Rusli --- .../_torch/speculative/hw_agnostic/test_dspark_eplb_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py index f70c98faf5cf..cbd3af719bee 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py @@ -75,6 +75,7 @@ def _model_config(lb_config=None, num_hidden_layers=NUM_HIDDEN_LAYERS): # ModelConfig always carries it, so the helper reads it unguarded. max_seq_len=8192, use_low_precision_moe_combine=False, + enable_wide_ep_fault_tolerance=False, megamoe_fuse_shared_expert=False, pretrained_config=SimpleNamespace(num_hidden_layers=num_hidden_layers), ) From 58b4e7e12756029bbf1159d5a776235df7ee9e07 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Thu, 19 Nov 2026 14:52:40 -0800 Subject: [PATCH 05/13] [None][feat] EPLB: unbacked spare expert slots and expert restore into them PR 2 of 7 of Wide-EP fault tolerance. After a rank fails, the survivors must serve its experts on the CUDA graphs they already captured, without paying GPU memory for that while healthy. - Adds unbacked spare EPLB slots: each rank's slot range is mapped at a fixed address, and spare slots map one shared scratch chunk until a restore backs them with memory of their own. - Adds the restore: every surviving rank plans the same new placement, backs the spare slots the plan fills, loads the lost experts from host weights, and installs the placement. - The MoE GEMM autotuner profiles only the active slots, so spare slots add no tuning workspace and change no tactic. - A MoE backend or quantization method that cannot run with spare slots is turned down at selection, with the reason. - Nothing creates spare slots before PR 6; with FT off, placement, routing, weight loading and tuning take main's paths. Signed-off-by: Andrew Rusli --- .../moe/loadBalance/moeLoadBalanceKernels.cu | 14 +- .../moe/loadBalance/moeLoadBalanceKernels.h | 11 +- .../nanobind/runtime/moeBindings.cpp | 28 +- cpp/tensorrt_llm/runtime/CMakeLists.txt | 8 +- .../hostAccessibleDeviceAllocator.cpp | 79 ++ .../hostAccessibleDeviceAllocator.h | 20 + .../moeLoadBalancer/moeLoadBalancer.cpp | 333 +++++++- .../runtime/moeLoadBalancer/moeLoadBalancer.h | 72 +- .../moeLoadBalancer/unbackedSlotMemory.cpp | 530 +++++++++++++ .../moeLoadBalancer/unbackedSlotMemory.h | 144 ++++ .../thop/moe/loadBalance/moeLoadBalanceOp.cpp | 3 +- .../moe/loadBalance/moeUnbackedSlotOp.cpp | 122 +++ .../kernels/moe/moeLoadBalanceKernelTest.cpp | 56 ++ cpp/tests/unit_tests/runtime/CMakeLists.txt | 2 + .../hostAccessibleDeviceAllocatorTest.cu | 138 +++- .../runtime/moeLoadBalancerTest.cpp | 729 +++++++++++++++++- .../_torch/custom_ops/cpp_custom_ops.py | 9 + .../_torch/custom_ops/cute_dsl_custom_ops.py | 128 ++- .../_torch/custom_ops/torch_custom_ops.py | 17 +- .../moe/fused_moe/MOE_DEVELOPER_GUIDE.md | 2 + .../_torch/moe/fused_moe/create_moe.py | 8 +- .../cutedsl/grouped_gemm/eligibility.py | 2 + .../fused_moe/cutedsl/grouped_gemm/nvfp4.py | 60 +- .../_torch/moe/fused_moe/cutedsl/runners.py | 10 + .../_torch/moe/fused_moe/cutlass/base.py | 11 +- .../moe/fused_moe/cutlass/eligibility.py | 2 + .../_torch/moe/fused_moe/cutlass/identity.py | 6 + .../_torch/moe/fused_moe/cutlass/nvfp4.py | 1 + .../moe/fused_moe/cutlass/unquantized.py | 1 + .../moe/fused_moe/eplb_expert_restore.py | 549 +++++++++++++ .../_torch/moe/fused_moe/impl_blocks.py | 56 +- .../_torch/moe/fused_moe/impl_contract.py | 29 + .../_torch/moe/fused_moe/interface.py | 15 +- .../_torch/moe/fused_moe/moe_load_balancer.py | 123 ++- .../_torch/moe/fused_moe/moe_resolution.py | 39 +- .../pyexecutor/rank_failure_model_load.py | 11 +- tensorrt_llm/_wide_ep_fault_tolerance.py | 18 + .../test_lists/test-db/l0_b200.yml | 2 + .../moe/kernels/parallel/test_cute_dsl_moe.py | 24 + tests/unittest/_torch/moe/test_moe_backend.py | 378 ++++++++- tests/unittest/_torch/moe/test_moe_impl.py | 82 ++ .../_torch/moe/test_moe_load_balancer.py | 452 ++++++++++- tests/unittest/_torch/moe/test_spare_slots.py | 412 ++++++++++ .../_torch/moe/test_w4a16_nvfp4_eplb.py | 4 +- 44 files changed, 4596 insertions(+), 144 deletions(-) create mode 100644 cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.cpp create mode 100644 cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h create mode 100644 cpp/tensorrt_llm/thop/moe/loadBalance/moeUnbackedSlotOp.cpp create mode 100644 tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py create mode 100644 tensorrt_llm/_wide_ep_fault_tolerance.py create mode 100644 tests/unittest/_torch/moe/test_spare_slots.py diff --git a/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.cu b/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.cu index 5efb9ef0c675..f7cedb6aefaa 100644 --- a/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.cu +++ b/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.cu @@ -624,7 +624,8 @@ __global__ void moeComputeRouteSortKernel(MoeLoadBalanceMetaInfo metaInfo, MoePl } void moeComputeRouteDevice(MoeLoadBalanceMetaInfo metaInfo, MoePlacementInfo placementInfo, - int* const tokenSelectedExperts, int* tokenRoutedSlotIds, int tokenCount, bool offsetByEpRank, cudaStream_t stream) + int* const tokenSelectedExperts, int* tokenRoutedSlotIds, int tokenCount, bool offsetByEpRank, cudaStream_t stream, + bool singleReplicaPerExpert) { constexpr int kThreadCount = 256; constexpr int kEltPerThread = 4; @@ -635,7 +636,9 @@ void moeComputeRouteDevice(MoeLoadBalanceMetaInfo metaInfo, MoePlacementInfo pla return; } int dynamicShmSize = sizeof(int16_t) * metaInfo.epSize * metaInfo.slotCountPerRank; - if (metaInfo.expertCount == metaInfo.epSize * metaInfo.slotCountPerRank) + // With a single replica per expert, expertReplicaStartOffset[e] is e, so globalSlotIds[e] is expert e's only slot. + // The direct kernel is then correct even when empty spare slots make the slots outnumber the experts. + if (singleReplicaPerExpert || metaInfo.expertCount == metaInfo.epSize * metaInfo.slotCountPerRank) { auto* kernelFn = moeComputeRouteNoRedundantKernel<1024, kThreadCount, kEltPerThread>; // no redundant expert, so we don't need complex routing, but just assign to the correct solt. @@ -650,15 +653,20 @@ void moeComputeRouteDevice(MoeLoadBalanceMetaInfo metaInfo, MoePlacementInfo pla } } -void moeWaitSignalForCpuStageHost(MoeLoadBalanceSingleLayerSignal* signal) +bool moeWaitSignalForCpuStageHost(MoeLoadBalanceSingleLayerSignal* signal, std::atomic const* stop) { bool ready = false; do { auto loaded = __atomic_load_n(&signal->stepAndOwner, __ATOMIC_ACQUIRE); ready = getOwnerDevice(loaded) == MoeLoadBalanceSingleLayerSignal::kCPU; + if (!ready && stop != nullptr && stop->load(std::memory_order_relaxed)) + { + return false; + } } while (!ready); std::atomic_thread_fence(std::memory_order_acquire); + return true; } void moeSetSignalForGpuStageHost(MoeLoadBalanceSingleLayerSignal* signal, int64_t iterId, bool enableStatistic) diff --git a/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h b/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h index 11475b139962..6bee0812c0cd 100644 --- a/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h +++ b/cpp/tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h @@ -19,6 +19,8 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceCommon.h" +#include + TRTLLM_NAMESPACE_BEGIN namespace kernels @@ -110,8 +112,11 @@ void moeHierarchicalStatisticUpdate(MoeLoadBalanceMetaInfo metaInfo, MoeLoadBala // @param tokenCount: the token count to compute the route // @param offsetByEpRank: whether to offset the round robin position by epRank // @param stream: the CUDA stream to be used +// @param singleReplicaPerExpert: whether every expert has a single replica, though empty spare slots may make the slots +// outnumber the experts. Selects the direct expert-to-slot kernel. void moeComputeRouteDevice(MoeLoadBalanceMetaInfo metaInfo, MoePlacementInfo placementInfo, - int* const tokenSelectedExperts, int* tokenRoutedSlotIds, int tokenCount, bool offsetByEpRank, cudaStream_t stream); + int* const tokenSelectedExperts, int* tokenRoutedSlotIds, int tokenCount, bool offsetByEpRank, cudaStream_t stream, + bool singleReplicaPerExpert = false); // @brief wait for the signal from gpu to cpu on host // @@ -120,7 +125,9 @@ void moeComputeRouteDevice(MoeLoadBalanceMetaInfo metaInfo, MoePlacementInfo pla // After this function is called, functions for weight update can be called. // // @param signal: the signal -void moeWaitSignalForCpuStageHost(MoeLoadBalanceSingleLayerSignal* signal); +// @param stop: optional flag that ends the wait once set +// @return: false if the wait was stopped +bool moeWaitSignalForCpuStageHost(MoeLoadBalanceSingleLayerSignal* signal, std::atomic const* stop = nullptr); // @brief set the signal for gpu stage on host // diff --git a/cpp/tensorrt_llm/nanobind/runtime/moeBindings.cpp b/cpp/tensorrt_llm/nanobind/runtime/moeBindings.cpp index 5f0982eeeb60..7672c2ae47db 100644 --- a/cpp/tensorrt_llm/nanobind/runtime/moeBindings.cpp +++ b/cpp/tensorrt_llm/nanobind/runtime/moeBindings.cpp @@ -95,12 +95,16 @@ void initMoeBindings(nb::module_& m) .def("get_layer_id", &tr::SingleLayerMoeLoadBalancer::getLayerId, "Get the layer id of the SingleLayerMoeLoadBalancer", nb::call_guard()) .def("get_old_rank_expert_ids", &tr::SingleLayerMoeLoadBalancer::getOldRankExpertIds, - "Get the old rank expert ids of the SingleLayerMoeLoadBalancer", nb::call_guard()); + "Get the old rank expert ids of the SingleLayerMoeLoadBalancer", nb::call_guard()) + .def("get_unusable_local_slots", &tr::SingleLayerMoeLoadBalancer::getUnusableLocalSlots, + "Get the local slots that EPLB updates leave empty", nb::call_guard()); // Bind MoeLoadBalancer class nb::class_(m, "MoeLoadBalancer") - .def(nb::init(), nb::arg("ep_rank"), nb::arg("ep_size"), nb::arg("layer_updates_per_iter"), - "Initialize the MoeLoadBalancer with the specified expert parallel rank, size, and update frequency", + .def(nb::init(), nb::arg("ep_rank"), nb::arg("ep_size"), nb::arg("layer_updates_per_iter"), + nb::arg("fault_tolerance") = false, + "Initialize the MoeLoadBalancer with the specified expert parallel rank, size, update frequency and " + "whether Wide-EP fault tolerance is enabled", nb::call_guard()) .def("set_use_gpu_memcpy", &tr::MoeLoadBalancer::setUseGpuMemcpy, nb::arg("use_gpu_memcpy"), "Set whether to use GPU memcpy for weight updates", nb::call_guard()) @@ -120,11 +124,29 @@ void initMoeBindings(nb::module_& m) .def("reconfigure_mask_only", &tr::MoeLoadBalancer::reconfigureMaskOnly, nb::arg("dead_ranks"), "Reconfigure EPLB routing metadata so slots on dead EP ranks are unreachable", nb::call_guard()) + .def("restore_placement", &tr::MoeLoadBalancer::restorePlacement, nb::arg("dead_ranks"), + nb::arg("layer_rank_expert_ids"), + "Under Wide-EP fault tolerance, set the failed EP ranks and install a placement per layer whose changed " + "local slots the caller loaded", + nb::call_guard()) + .def("abandon_iterations", &tr::MoeLoadBalancer::abandonIterations, + "Under Wide-EP fault tolerance, after a rank failure interrupted the forward, end the online EPLB work of " + "the iterations started so far and restart the statistics and the update plan", + nb::call_guard()) + .def("check_slots_writable", &tr::MoeLoadBalancer::checkSlotsWritable, + "Under Wide-EP fault tolerance, raise unless no iteration runs or is queued and the online EPLB worker is " + "done with the last one", + nb::call_guard()) .def("shutdown", &tr::MoeLoadBalancer::shutdown, "Shutdown the load balancer and clean up resources", nb::call_guard()); m.def("is_host_accessible_device_memory_supported", &tr::HostAccessibleDeviceAllocator::isSupported, "If current system support host accessible device memory"); + m.def( + "host_accessible_device_memory_uses_gdrcopy", + [] { return tr::HostAccessibleDeviceAllocator::getInstance().usesGdrCopy(); }, + "Whether host accessible device memory is device memory that GDRCopy maps for the host, which spare EPLB " + "slots of online EPLB need under Wide-EP fault tolerance; only while a load balancer exists"); // Bind do_replication function for testing m.def("do_replication", &pyDoReplication, nb::arg("meta_info"), nb::arg("expert_load_factor"), diff --git a/cpp/tensorrt_llm/runtime/CMakeLists.txt b/cpp/tensorrt_llm/runtime/CMakeLists.txt index 3b89d8f414b8..82fe88d427f8 100644 --- a/cpp/tensorrt_llm/runtime/CMakeLists.txt +++ b/cpp/tensorrt_llm/runtime/CMakeLists.txt @@ -15,8 +15,12 @@ include(FetchContent) set(SRCS - bufferManagerNvls.cpp utils/debugUtils.cu ipcUtils.cpp - mcastDeviceMemory.cpp moeLoadBalancer/moeLoadBalancer.cpp + bufferManagerNvls.cpp + utils/debugUtils.cu + ipcUtils.cpp + mcastDeviceMemory.cpp + moeLoadBalancer/moeLoadBalancer.cpp + moeLoadBalancer/unbackedSlotMemory.cpp ncclCommunicator.cpp) include_directories(${API_INCLUDE_DIR}/tensorrt_llm/runtime) diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.cpp b/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.cpp index 804d2916170c..638e95ce3322 100644 --- a/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.cpp +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.cpp @@ -374,6 +374,85 @@ void HostAccessibleDeviceAllocator::memcpyToDevice(void* dst, void const* src, s } } +void HostAccessibleDeviceAllocator::memcpyToDeviceAddress(void* devDst, void const* src, size_t size) +{ + if (mGdrHandle == nullptr) + { + // The host accesses the memory at its device address. + memcpy(devDst, src, size); + return; + } + auto* dst = static_cast(devDst); + auto const* from = static_cast(src); + while (size > 0) + { + auto const allocationInfo = getAllocationInfoFromDevPtr(dst); + auto* const begin = static_cast(allocationInfo.devPtr); + TLLM_CHECK_WITH_INFO(allocationInfo.memDesc != nullptr && dst < begin + allocationInfo.size, + "%p is not host accessible device memory", static_cast(dst)); + size_t const part = std::min(size, static_cast(begin + allocationInfo.size - dst)); + // Like memcpyToDevice, rely on the range check above rather than the return code. + tensorrt_llm::runtime::gdrcopy::copy_to_mapping( + allocationInfo.memDesc->gdrMh, static_cast(allocationInfo.hostPtr) + (dst - begin), from, part); + dst += part; + from += part; + size -= part; + } +} + +bool HostAccessibleDeviceAllocator::usesGdrCopy() const +{ + TLLM_CHECK_WITH_INFO(mIsInited, "Host accessible device memory is chosen once a load balancer exists"); + return mGdrHandle != nullptr; +} + +void HostAccessibleDeviceAllocator::mapDeviceMemory(void* devPtr, size_t size) +{ + TLLM_CHECK_WITH_INFO(mGdrHandle != nullptr, "Mapping device memory for the host needs GDRCopy"); + auto const address = reinterpret_cast(devPtr); + TLLM_CHECK_WITH_INFO(size > 0 && address % GPU_PAGE_SIZE == 0 && size % GPU_PAGE_SIZE == 0, + "GDRCopy maps whole GPU pages, not %p (%zu bytes)", devPtr, size); + gdr_mh_t mh{}; + TLLM_CHECK_WITH_INFO(tensorrt_llm::runtime::gdrcopy::pin_buffer(mGdrHandle, address, size, 0, 0, &mh) == 0, + "GDR pin_buffer failed for %p (%zu bytes)", devPtr, size); + void* map = nullptr; + if (tensorrt_llm::runtime::gdrcopy::map(mGdrHandle, mh, &map, size) != 0) + { + tensorrt_llm::runtime::gdrcopy::unpin_buffer(mGdrHandle, mh); + TLLM_THROW("GDR map failed for %p (%zu bytes)", devPtr, size); + } + // The descriptor owns no device memory, so free() would release only the host mapping, never the range. + auto* memDesc = new gdrcopy::GdrMemDesc{nullptr, map, 0, size, mh}; + recordAllocation(devPtr, size, map, memDesc); +} + +void HostAccessibleDeviceAllocator::unmapDeviceMemory(void* devPtr) noexcept +{ + gdrcopy::GdrMemDesc* memDesc = nullptr; + { + std::unique_lock lock(mAllocationsMutex); + auto it = mDeviceAllocations.find(devPtr); + if (it == mDeviceAllocations.end() || it->second.memDesc == nullptr + || it->second.memDesc->gdrDeviceMem != nullptr) + { + TLLM_LOG_ERROR("%p is not device memory that mapDeviceMemory mapped", devPtr); + return; + } + memDesc = it->second.memDesc; + mHostAllocations.erase(it->second.hostPtr); + mDeviceAllocations.erase(it); + } + if (tensorrt_llm::runtime::gdrcopy::unmap(mGdrHandle, memDesc->gdrMh, memDesc->gdrMap, memDesc->gdrMapSize) != 0) + { + TLLM_LOG_ERROR("Unmapping device memory at %p for the host failed", devPtr); + } + if (tensorrt_llm::runtime::gdrcopy::unpin_buffer(mGdrHandle, memDesc->gdrMh) != 0) + { + TLLM_LOG_ERROR("Unpinning device memory at %p failed", devPtr); + } + delete memDesc; +} + void* HostAccessibleDeviceAllocator::allocate(size_t memorySize) { int currentDevId = -1; diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h b/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h index be921d4fb82d..dc310a3f3322 100644 --- a/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h @@ -87,6 +87,26 @@ class HostAccessibleDeviceAllocator */ void memcpyToDevice(void* dst, void const* src, size_t size); + /** + * @brief Copy from the host to host accessible device memory at a device address, across allocations if needed. + */ + void memcpyToDeviceAddress(void* devDst, void const* src, size_t size); + + /** + * @brief Return whether host accessible device memory is device memory that GDRCopy maps for the host. + */ + bool usesGdrCopy() const; + + /** + * @brief Pin and map GPU-page aligned device memory of one allocation for the host with GDRCopy. + */ + void mapDeviceMemory(void* devPtr, size_t size); + + /** + * @brief Unmap and unpin device memory that mapDeviceMemory mapped, logging errors. + */ + void unmapDeviceMemory(void* devPtr) noexcept; + private: struct AllocationInfo { diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.cpp b/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.cpp index 2047c28654e1..3cf9462bead8 100644 --- a/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.cpp +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h" #include "topologyDetector.h" +#include "unbackedSlotMemory.h" #include #include #include @@ -62,31 +63,51 @@ bool isRankMasked(std::vector const* deadRankMask, int rank) return deadRankMask != nullptr && ((*deadRankMask)[rank] != 0); } -int getActiveSlotCount( - tensorrt_llm::kernels::MoeLoadBalanceMetaInfo const& metaInfo, std::vector const* deadRankMask) +bool isSlotUsable(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo const& metaInfo, + std::vector const* deadRankMask, std::vector const* unusableSlotMask, int rank, int localSlotId) +{ + return !isRankMasked(deadRankMask, rank) + && (unusableSlotMask == nullptr || (*unusableSlotMask)[rank * metaInfo.slotCountPerRank + localSlotId] == 0); +} + +std::vector getUsableSlotCounts(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo const& metaInfo, + std::vector const* deadRankMask, std::vector const* unusableSlotMask) { TLLM_CHECK_WITH_INFO(deadRankMask == nullptr || static_cast(deadRankMask->size()) == metaInfo.epSize, "deadRankMask size (%ld) must match epSize (%d)", deadRankMask == nullptr ? 0L : static_cast(deadRankMask->size()), metaInfo.epSize); + TLLM_CHECK_WITH_INFO(unusableSlotMask == nullptr + || static_cast(unusableSlotMask->size()) == metaInfo.epSize * metaInfo.slotCountPerRank, + "unusableSlotMask size (%ld) must match the slot count (%d)", + unusableSlotMask == nullptr ? 0L : static_cast(unusableSlotMask->size()), + metaInfo.epSize * metaInfo.slotCountPerRank); - int activeRankCount = 0; + std::vector usableSlotCounts(metaInfo.epSize, 0); for (int rank = 0; rank < metaInfo.epSize; ++rank) { - if (!isRankMasked(deadRankMask, rank)) + if (unusableSlotMask == nullptr) { - ++activeRankCount; + usableSlotCounts[rank] = isRankMasked(deadRankMask, rank) ? 0 : metaInfo.slotCountPerRank; + continue; + } + for (int localSlotId = 0; localSlotId < metaInfo.slotCountPerRank; ++localSlotId) + { + usableSlotCounts[rank] + += static_cast(isSlotUsable(metaInfo, deadRankMask, unusableSlotMask, rank, localSlotId)); } } - return activeRankCount * metaInfo.slotCountPerRank; + return usableSlotCounts; } } // namespace void doReplication(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* const expertLoadFactor, - MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask) + MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask, + std::vector const* unusableSlotMask) { cpuPlacement->expertReplicaCount.resize(metaInfo.expertCount); - int totalSlotCount = getActiveSlotCount(metaInfo, deadRankMask); + auto const usableSlotCounts = getUsableSlotCounts(metaInfo, deadRankMask, unusableSlotMask); + int totalSlotCount = std::accumulate(usableSlotCounts.begin(), usableSlotCounts.end(), 0); TLLM_CHECK_WITH_INFO(totalSlotCount >= metaInfo.expertCount, "Mask-only EPLB reconfigure would leave fewer active slots (%d) than experts (%d)", totalSlotCount, metaInfo.expertCount); @@ -153,13 +174,15 @@ void doReplication(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float } void doPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* const expertLoadFactor, - MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask) + MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask, + std::vector const* unusableSlotMask) { // This function only update these two vectors auto& rankExpertIds = cpuPlacement->rankExpertIds; auto& replicaCount = cpuPlacement->expertReplicaCount; - int totalSlotCount = getActiveSlotCount(metaInfo, deadRankMask); + auto const usableSlotCounts = getUsableSlotCounts(metaInfo, deadRankMask, unusableSlotMask); + int totalSlotCount = std::accumulate(usableSlotCounts.begin(), usableSlotCounts.end(), 0); // 1. Create all replica information std::vector allReplicas; allReplicas.reserve(totalSlotCount); @@ -186,6 +209,7 @@ void doPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* // 3. Maintain Rank state and initialize Priority Queue std::vector currentRankLoad(metaInfo.epSize, 0.0); std::vector currentRankSlots(metaInfo.epSize, 0); // Tracks the count of assigned slots per rank + std::vector nextLocalSlotId(metaInfo.epSize, 0); for (int rank = 0; rank < metaInfo.epSize; ++rank) { std::fill_n(rankExpertIds[rank].begin(), metaInfo.slotCountPerRank, -1); @@ -198,7 +222,7 @@ void doPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* // Initialize the priority queue with all ranks having 0 load for (int rank = 0; rank < metaInfo.epSize; ++rank) { - if (!isRankMasked(deadRankMask, rank)) + if (usableSlotCounts[rank] > 0) { pq.push({0.0, rank}); } @@ -215,17 +239,23 @@ void doPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* int bestRank = top.second; double currentLoad = top.first; // The load before adding this replica - int localSlotId = currentRankSlots[bestRank]; + int localSlotId = nextLocalSlotId[bestRank]; + while (localSlotId < metaInfo.slotCountPerRank + && !isSlotUsable(metaInfo, deadRankMask, unusableSlotMask, bestRank, localSlotId)) + { + ++localSlotId; + } TLLM_CHECK_WITH_INFO(localSlotId < metaInfo.slotCountPerRank, "localSlotId=%d should be less than metaInfo.slotCountPerRank=%d", localSlotId, metaInfo.slotCountPerRank); rankExpertIds[bestRank][localSlotId] = replica.expertId; + nextLocalSlotId[bestRank] = localSlotId + 1; // Update rank state currentRankLoad[bestRank] = currentLoad + replica.slotSize; // Update load explicitly currentRankSlots[bestRank]++; // Increment the slot count for this rank // If the rank still has capacity, push it back into the queue with updated load - if (currentRankSlots[bestRank] < metaInfo.slotCountPerRank) + if (currentRankSlots[bestRank] < usableSlotCounts[bestRank]) { pq.push({currentRankLoad[bestRank], bestRank}); } @@ -459,7 +489,8 @@ SingleLayerMoeLoadBalancer::~SingleLayerMoeLoadBalancer() void SingleLayerMoeLoadBalancer::addSingleWeightSlot(int localSlotId, std::string const& name, MoeWeight weightSlot) { - mWeightUpdater->addSingleWeightSlot(localSlotId, name, weightSlot, mMoeLoadBalancer->mUseGpuMemcpy); + mWeightUpdater->addSingleWeightSlot( + localSlotId, name, weightSlot, mMoeLoadBalancer->mUseGpuMemcpy, mMoeLoadBalancer->mFaultTolerance); } void SingleLayerMoeLoadBalancer::addSingleHostWeight(int expertId, std::string const& name, MoeWeight hostWeight) @@ -470,16 +501,27 @@ void SingleLayerMoeLoadBalancer::addSingleHostWeight(int expertId, std::string c void SingleLayerMoeLoadBalancer::setInitialWeightAssignments(std::vector const& initialWeightAssignments) { std::fill_n(mCpuPlacementInfo.expertReplicaCount.begin(), mMetaInfo.expertCount, 0); + mUnusableSlots.clear(); + // Only fault tolerance's spare slots start empty (-1). + int const minExpertId = mMoeLoadBalancer->mFaultTolerance ? -1 : 0; for (int rank = 0; rank < mMetaInfo.epSize; ++rank) { for (int localSlotId = 0; localSlotId < mMetaInfo.slotCountPerRank; ++localSlotId) { - int expertId = initialWeightAssignments[rank * mMetaInfo.slotCountPerRank + localSlotId]; - TLLM_CHECK_WITH_INFO(expertId >= 0 && expertId < mMetaInfo.expertCount, "expertId=%d", expertId); + int const globalSlotId = rank * mMetaInfo.slotCountPerRank + localSlotId; + int expertId = initialWeightAssignments[globalSlotId]; + TLLM_CHECK_WITH_INFO(expertId >= minExpertId && expertId < mMetaInfo.expertCount, "expertId=%d", expertId); mCpuPlacementInfo.rankExpertIds[rank][localSlotId] = expertId; + if (expertId == -1) + { + mUnusableSlots.resize(mMetaInfo.epSize * mMetaInfo.slotCountPerRank, 0); + mUnusableSlots[globalSlotId] = 1; + continue; + } mCpuPlacementInfo.expertReplicaCount[expertId]++; } } + updateSingleReplicaPerExpert(); prepareGpuPlacementInfo(mMetaInfo, &mCpuPlacementInfo); // we don't need to call mWeightUpdater since should be already assigned. copyPlacementInfoToGpu(); @@ -542,7 +584,20 @@ void SingleLayerMoeLoadBalancer::setGpuStage() void SingleLayerMoeLoadBalancer::waitCpuStage() { - tensorrt_llm::kernels::moeWaitSignalForCpuStageHost(mSingleLayerSignal); + // An abandoned iteration may never hand the layer back, so no update starts. + auto const* stop = mMoeLoadBalancer->mFaultTolerance ? &mMoeLoadBalancer->mAbandoning : nullptr; + if (!tensorrt_llm::kernels::moeWaitSignalForCpuStageHost(mSingleLayerSignal, stop)) + { + mUpdateWeightsEnabled = false; + } +} + +void SingleLayerMoeLoadBalancer::resetAfterAbandon() +{ + std::fill_n(mStatisticInfo.expertLoadFactor, mMetaInfo.expertCount, 0.0f); + auto const stepAndOwner = __atomic_load_n(&mSingleLayerSignal->stepAndOwner, __ATOMIC_ACQUIRE); + __atomic_store_n(&mSingleLayerSignal->stepAndOwner, + stepAndOwner | tensorrt_llm::kernels::MoeLoadBalanceSingleLayerSignal::kCPU, __ATOMIC_RELEASE); } void SingleLayerMoeLoadBalancer::maybeStartUpdateWeights() @@ -576,17 +631,28 @@ void SingleLayerMoeLoadBalancer::waitLastUpdateDone() std::vector SingleLayerMoeLoadBalancer::computeMaskOnlyReplicaCounts( std::vector const& deadRankMask) const +{ + return computeReplicaCounts(deadRankMask, mCpuPlacementInfo.oldRankExpertIds, "Mask-only EPLB reconfigure"); +} + +std::vector SingleLayerMoeLoadBalancer::computeReplicaCounts(std::vector const& deadRankMask, + std::vector> const& rankExpertIds, char const* operation) const { TLLM_CHECK_WITH_INFO(static_cast(deadRankMask.size()) == mMetaInfo.epSize, "deadRankMask size (%ld) must match epSize (%d)", static_cast(deadRankMask.size()), mMetaInfo.epSize); + TLLM_CHECK_WITH_INFO(static_cast(rankExpertIds.size()) == mMetaInfo.epSize, + "placement has %ld ranks, expected %d", static_cast(rankExpertIds.size()), mMetaInfo.epSize); std::vector expertReplicaCount(mMetaInfo.expertCount, 0); for (int rank = 0; rank < mMetaInfo.epSize; ++rank) { bool const rankDead = deadRankMask[rank] != 0; + TLLM_CHECK_WITH_INFO(static_cast(rankExpertIds[rank].size()) == mMetaInfo.slotCountPerRank, + "placement of rank %d has %ld slots, expected %d", rank, static_cast(rankExpertIds[rank].size()), + mMetaInfo.slotCountPerRank); for (int localSlotId = 0; localSlotId < mMetaInfo.slotCountPerRank; ++localSlotId) { - int expertId = mCpuPlacementInfo.oldRankExpertIds[rank][localSlotId]; + int expertId = rankExpertIds[rank][localSlotId]; TLLM_CHECK_WITH_INFO(expertId == -1 || (expertId >= 0 && expertId < mMetaInfo.expertCount), "expertId=%d should be -1 or in range [0, %d)", expertId, mMetaInfo.expertCount); if (!rankDead && expertId >= 0) @@ -598,15 +664,34 @@ std::vector SingleLayerMoeLoadBalancer::computeMaskOnlyReplicaCounts( for (int expertId = 0; expertId < mMetaInfo.expertCount; ++expertId) { - TLLM_CHECK_WITH_INFO(expertReplicaCount[expertId] > 0, - "Mask-only EPLB reconfigure would leave expert %d with no surviving replica", expertId); + TLLM_CHECK_WITH_INFO(expertReplicaCount[expertId] > 0, "%s would leave expert %d with no surviving replica", + operation, expertId); } return expertReplicaCount; } +bool SingleLayerMoeLoadBalancer::usesSingleReplicaRouting() const +{ + return mMoeLoadBalancer->mFaultTolerance && mSingleReplicaPerExpert; +} + +void SingleLayerMoeLoadBalancer::updateSingleReplicaPerExpert() +{ + auto const& counts = mCpuPlacementInfo.expertReplicaCount; + mSingleReplicaPerExpert = std::all_of(counts.begin(), counts.end(), [](int count) { return count == 1; }); +} + +void SingleLayerMoeLoadBalancer::checkKeepsSingleReplicaPerExpert(std::vector const& expertReplicaCount) const +{ + TLLM_CHECK_WITH_INFO(!usesSingleReplicaRouting() + || std::all_of(expertReplicaCount.begin(), expertReplicaCount.end(), [](int count) { return count == 1; }), + "Routing of layer %d uses the single-replica kernel, so a placement must keep a single replica per expert", + mLayerId); +} + void SingleLayerMoeLoadBalancer::validateMaskOnly(std::vector const& deadRankMask) const { - (void) computeMaskOnlyReplicaCounts(deadRankMask); + checkKeepsSingleReplicaPerExpert(computeMaskOnlyReplicaCounts(deadRankMask)); } void SingleLayerMoeLoadBalancer::reconfigureMaskOnly(std::vector const& deadRankMask) @@ -629,6 +714,51 @@ void SingleLayerMoeLoadBalancer::reconfigureMaskOnly(std::vector const& TLLM_CUDA_CHECK(cudaEventSynchronize(mUpdateWeightsDoneEvent)); } +void SingleLayerMoeLoadBalancer::validatePlacement( + std::vector const& deadRankMask, std::vector> const& rankExpertIds) const +{ + checkKeepsSingleReplicaPerExpert(computeReplicaCounts(deadRankMask, rankExpertIds, "EPLB placement")); +} + +void SingleLayerMoeLoadBalancer::installPlacement( + std::vector const& deadRankMask, std::vector> const& rankExpertIds) +{ + for (int rank = 0; rank < mMetaInfo.epSize; ++rank) + { + for (int localSlotId = 0; localSlotId < mMetaInfo.slotCountPerRank; ++localSlotId) + { + bool const empty = rankExpertIds[rank][localSlotId] == -1; + if (deadRankMask[rank] != 0 || (!empty && mUnusableSlots.empty())) + { + continue; + } + mUnusableSlots.resize(mMetaInfo.epSize * mMetaInfo.slotCountPerRank, 0); + mUnusableSlots[rank * mMetaInfo.slotCountPerRank + localSlotId] = static_cast(empty); + } + } + mCpuPlacementInfo.rankExpertIds = rankExpertIds; + mCpuPlacementInfo.oldRankExpertIds = rankExpertIds; + reconfigureMaskOnly(deadRankMask); +} + +std::vector SingleLayerMoeLoadBalancer::getUnusableLocalSlots() const +{ + std::vector localSlotIds; + if (mUnusableSlots.empty()) + { + return localSlotIds; + } + auto const rankSlots = mUnusableSlots.begin() + mMetaInfo.epRank * mMetaInfo.slotCountPerRank; + for (int localSlotId = 0; localSlotId < mMetaInfo.slotCountPerRank; ++localSlotId) + { + if (rankSlots[localSlotId] != 0) + { + localSlotIds.push_back(localSlotId); + } + } + return localSlotIds; +} + cudaStream_t SingleLayerMoeLoadBalancer::getStream() const { return mMoeLoadBalancer->mStream; @@ -676,8 +806,9 @@ void SingleLayerMoeLoadBalancer::copyPlacementInfoToGpuByCpu() void SingleLayerMoeLoadBalancer::updateWeightsRoutine() { auto const deadRankMask = mMoeLoadBalancer->getDeadRankMaskSnapshot(); - doReplication(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask); - doPlacement(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask); + auto const* unusableSlotMask = mUnusableSlots.empty() ? nullptr : &mUnusableSlots; + doReplication(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask, unusableSlotMask); + doPlacement(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask, unusableSlotMask); prepareGpuPlacementInfo(mMetaInfo, &mCpuPlacementInfo); mWeightUpdater->updateWeights(&mCpuPlacementInfo); copyPlacementInfoToGpu(); @@ -691,8 +822,9 @@ void SingleLayerMoeLoadBalancer::updateWeightsRoutine() void SingleLayerMoeLoadBalancer::updateWeightsRoutineByCpu() { auto const deadRankMask = mMoeLoadBalancer->getDeadRankMaskSnapshot(); - doReplication(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask); - doPlacement(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask); + auto const* unusableSlotMask = mUnusableSlots.empty() ? nullptr : &mUnusableSlots; + doReplication(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask, unusableSlotMask); + doPlacement(mMetaInfo, mStatisticInfo.expertLoadFactor, &mCpuPlacementInfo, &deadRankMask, unusableSlotMask); prepareGpuPlacementInfo(mMetaInfo, &mCpuPlacementInfo); mLastUpdateTaskId = mMoeLoadBalancer->addCopyTask( [this](int rank, int size) { mWeightUpdater->updateWeights(&mCpuPlacementInfo, rank, size); }); @@ -715,8 +847,8 @@ MoeWeightUpdaterBase::MoeWeightUpdaterBase( { } -void MoeWeightUpdaterBase::addSingleWeightSlot( - int localSlotId, std::string const& name, tensorrt_llm::runtime::MoeWeight weightSlot, bool gpuAccess) +void MoeWeightUpdaterBase::addSingleWeightSlot(int localSlotId, std::string const& name, + tensorrt_llm::runtime::MoeWeight weightSlot, bool gpuAccess, bool faultTolerance) { TLLM_CHECK_WITH_INFO(mWeightSlotsFinalized == false, "Cannot add slots after finalize"); TLLM_CHECK_WITH_INFO(localSlotId >= 0 && localSlotId < mMetaInfo.slotCountPerRank, @@ -733,9 +865,17 @@ void MoeWeightUpdaterBase::addSingleWeightSlot( "localSlotId=%d, name=%s already added.", localSlotId, name.c_str()); if (!gpuAccess) { - // if using cpu copy, change to host accessible pointer. - weightSlot.mWeightPtr + // A CPU-copied slot outside host accessible memory counts as not added. + void* const hostPtr = tensorrt_llm::runtime::HostAccessibleDeviceAllocator::getInstance().getHostPtr(weightSlot.mWeightPtr); + if (!faultTolerance) + { + weightSlot.mWeightPtr = hostPtr; + } + else if (hostPtr == nullptr && !UnbackedSlotMemory::getInstance().isUnbacked(weightSlot.mWeightPtr)) + { + weightSlot.mWeightPtr = nullptr; + } } mWeightSlots[name][localSlotId] = weightSlot; } @@ -855,10 +995,23 @@ void HostMemoryMoeWeightUpdater::copyWeights(MoeWeight const& src, MoeWeight con } } -void HostMemoryMoeWeightUpdater::copyWeightsCpu(MoeWeight const& src, MoeWeight const& dst, int rank, int size) +void HostMemoryMoeWeightUpdater::copyWeightsCpu( + MoeWeight const& src, MoeWeight const& dst, int rank, int size, bool dstIsDeviceAddress) { TLLM_CHECK(src.mWeightPtr != nullptr && dst.mWeightPtr != nullptr); TLLM_CHECK(src.mHeight == dst.mHeight && src.mWidth == dst.mWidth); + auto& allocator = HostAccessibleDeviceAllocator::getInstance(); + auto const copy = [&allocator, dstIsDeviceAddress](char* to, char const* from, size_t bytes) + { + if (dstIsDeviceAddress) + { + allocator.memcpyToDeviceAddress(to, from, bytes); + } + else + { + allocator.memcpyToDevice(to, from, bytes); + } + }; char* srcPtr = static_cast(src.mWeightPtr); char* dstPtr = static_cast(dst.mWeightPtr); size_t singleCopySize, copyCount, srcPitch, dstPitch; @@ -880,16 +1033,14 @@ void HostMemoryMoeWeightUpdater::copyWeightsCpu(MoeWeight const& src, MoeWeight size_t threadCopyCount = fullCopyCount / size; for (size_t i = rank * threadCopyCount; i < (rank + 1) * threadCopyCount; i++) { - HostAccessibleDeviceAllocator::getInstance().memcpyToDevice( - dstPtr + i * dstPitch, srcPtr + i * srcPitch, singleCopySize); + copy(dstPtr + i * dstPitch, srcPtr + i * srcPitch, singleCopySize); } size_t threadStartOffset = rank * singleCopySize / size; size_t threadEndOffset = (rank + 1) * singleCopySize / size; size_t threadCopySize = threadEndOffset - threadStartOffset; for (size_t i = fullCopyCount; i < copyCount && threadCopySize > 0; i++) { - HostAccessibleDeviceAllocator::getInstance().memcpyToDevice( - dstPtr + i * dstPitch + threadStartOffset, srcPtr + i * srcPitch + threadStartOffset, threadCopySize); + copy(dstPtr + i * dstPitch + threadStartOffset, srcPtr + i * srcPitch + threadStartOffset, threadCopySize); } } @@ -912,15 +1063,17 @@ void HostMemoryMoeWeightUpdater::updateWeights( { // PrintUpdateInfo(mMetaInfo, placementCpuInfo); bool useGpu = mLayerLoadBalancer->mMoeLoadBalancer->mUseGpuMemcpy; + // Only fault tolerance's slots may hold no expert (-1). + int const minExpertId = mLayerLoadBalancer->mMoeLoadBalancer->mFaultTolerance ? -1 : 0; for (int slotId = 0; slotId < mMetaInfo.slotCountPerRank; ++slotId) { int oldExpertId = placementCpuInfo->oldRankExpertIds[mMetaInfo.epRank][slotId]; int newExpertId = placementCpuInfo->rankExpertIds[mMetaInfo.epRank][slotId]; - TLLM_CHECK_WITH_INFO(oldExpertId >= 0 && oldExpertId < mMetaInfo.expertCount, - "oldExpertId=%d, should in range [0, %d)", oldExpertId, mMetaInfo.expertCount); - TLLM_CHECK_WITH_INFO(newExpertId >= 0 && newExpertId < mMetaInfo.expertCount, - "newExpertId=%d, should in range [0, %d)", newExpertId, mMetaInfo.expertCount); - if (oldExpertId == newExpertId) + TLLM_CHECK_WITH_INFO(oldExpertId >= minExpertId && oldExpertId < mMetaInfo.expertCount, + "oldExpertId=%d, should in range [%d, %d)", oldExpertId, minExpertId, mMetaInfo.expertCount); + TLLM_CHECK_WITH_INFO(newExpertId >= minExpertId && newExpertId < mMetaInfo.expertCount, + "newExpertId=%d, should in range [%d, %d)", newExpertId, minExpertId, mMetaInfo.expertCount); + if (oldExpertId == newExpertId || newExpertId == -1) { continue; } @@ -935,7 +1088,8 @@ void HostMemoryMoeWeightUpdater::updateWeights( } else { - copyWeightsCpu(hostWeight, slotWeight, rank, size); + copyWeightsCpu( + hostWeight, slotWeight, rank, size, mLayerLoadBalancer->mMoeLoadBalancer->mFaultTolerance); } } } @@ -945,8 +1099,9 @@ void HostMemoryMoeWeightUpdater::updateWeights( // Moe Load Balancer /////////////////////////////////////////////////////////////////////////////////////////////////// -MoeLoadBalancer::MoeLoadBalancer(int epRank, int epSize, int layerUpdatesPerIter) - : mEpRank{epRank} +MoeLoadBalancer::MoeLoadBalancer(int epRank, int epSize, int layerUpdatesPerIter, bool faultTolerance) + : mFaultTolerance{faultTolerance} + , mEpRank{epRank} , mEpSize{epSize} , mDeadRankMask(epSize, 0) , mLayerUpdatesPerIter{layerUpdatesPerIter} @@ -1140,6 +1295,94 @@ void MoeLoadBalancer::reconfigureMaskOnly(std::vector const& deadRanks) } } +void MoeLoadBalancer::checkRecoveryCall(char const* action) const +{ + TLLM_CHECK_WITH_INFO(mFaultTolerance, "Cannot %s without Wide-EP fault tolerance.", action); + TLLM_CHECK_WITH_INFO(mModelFinalized, "Model is not finalized, cannot %s.", action); + TLLM_CHECK_WITH_INFO(!mIterActive, "Cannot %s while an iteration is active.", action); + int device = -1; + TLLM_CUDA_CHECK(cudaGetDevice(&device)); + TLLM_CHECK_WITH_INFO(device == mCudaDeviceId, + "Cannot %s with device %d current: the load balancer is on device %d.", action, device, mCudaDeviceId); +} + +void MoeLoadBalancer::checkUpdatesEnded(char const* action) const +{ + checkRecoveryCall(action); + TLLM_CHECK_WITH_INFO(mIterInfoQueue.empty(), "Cannot %s while iterations are queued.", action); + // The worker walks the layers of a started iteration without the lock, and may start weight updates. + TLLM_CHECK_WITH_INFO(mLayerUpdatesPerIter == 0 || mWorkerDoneIterId == mIterId, + "Cannot %s before the online EPLB worker is done with the last iteration: call abandonIterations first.", + action); +} + +void MoeLoadBalancer::checkSlotsWritable() +{ + std::lock_guard workerLock(mWorkerThreadMutex); + checkUpdatesEnded("write slots"); +} + +void MoeLoadBalancer::restorePlacement( + std::vector const& deadRanks, std::vector>> const& layerRankExpertIds) +{ + std::lock_guard workerLock(mWorkerThreadMutex); + checkUpdatesEnded("restore placement"); + TLLM_CHECK_WITH_INFO(layerRankExpertIds.size() == mLayers.size(), "Got placements for %ld layers, expected %ld", + static_cast(layerRankExpertIds.size()), static_cast(mLayers.size())); + std::vector deadRankMask(mEpSize, 0); + for (int deadRank : deadRanks) + { + TLLM_CHECK_WITH_INFO( + deadRank >= 0 && deadRank < mEpSize, "deadRank=%d should be in range [0, %d)", deadRank, mEpSize); + deadRankMask[deadRank] = 1; + } + TLLM_CHECK_WITH_INFO(deadRankMask[mEpRank] == 0, "Local epRank (%d) cannot be a failed rank", mEpRank); + for (size_t i = 0; i < mLayers.size(); ++i) + { + mLayers[i]->validatePlacement(deadRankMask, layerRankExpertIds[i]); + } + for (auto& layer : mLayers) + { + layer->waitLastUpdateDone(); + } + // Work queued on any stream may still load the slots or read the placement tables that installPlacement rewrites. + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + for (size_t i = 0; i < mLayers.size(); ++i) + { + mLayers[i]->installPlacement(deadRankMask, layerRankExpertIds[i]); + } + std::lock_guard maskLock(mDeadRankMaskMutex); + mDeadRankMask = deadRankMask; +} + +void MoeLoadBalancer::abandonIterations() +{ + std::unique_lock workerLock(mWorkerThreadMutex); + checkRecoveryCall("abandon iterations"); + if (mLayerUpdatesPerIter == 0) + { + return; + } + workerLock.unlock(); + // Kernels that wait on the signals run on the callers' streams. The worker keeps handing them + // their layers until the device is idle, so no GPU work uses the signals after. + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + workerLock.lock(); + mAbandoning.store(true, std::memory_order_release); + mWorkerDoneCondition.wait(workerLock, [this] { return mWorkerDoneIterId == mIterId; }); + for (auto& layer : mLayers) + { + layer->waitLastUpdateDone(); + } + for (auto& layer : mLayers) + { + layer->resetAfterAbandon(); + } + mUpdateLayerQueue.clear(); + generateUpdatePlan(); + mAbandoning.store(false, std::memory_order_release); +} + void MoeLoadBalancer::workerThread() { TLLM_CUDA_CHECK(cudaSetDevice(mCudaDeviceId)); @@ -1178,6 +1421,12 @@ void MoeLoadBalancer::workerThread() mUpdateLayerQueue.pop_front(); mUpdateLayerQueue.push_back(currentUpdatedLayers); } + if (mFaultTolerance) + { + std::lock_guard lock(mWorkerThreadMutex); + mWorkerDoneIterId = iterId; + mWorkerDoneCondition.notify_all(); + } } for (auto& layer : mLayers) { diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.h b/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.h index e25edf4244c6..19329b0e4732 100644 --- a/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.h +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.h @@ -79,7 +79,10 @@ class MoeWeightUpdaterBase virtual ~MoeWeightUpdaterBase() {} - void addSingleWeightSlot(int localSlotId, std::string const& name, MoeWeight weightSlot, bool gpuAccess); + // A CPU-copied slot keeps its host pointer, or with fault tolerance its device address, since a spare slot gets its + // host mapping only when a restore backs it. + void addSingleWeightSlot( + int localSlotId, std::string const& name, MoeWeight weightSlot, bool gpuAccess, bool faultTolerance); virtual void addSingleHostWeight(int expertId, std::string const& name, MoeWeight hostWeight) = 0; virtual void finalizeWeights(); virtual void updateWeights(MoePlacementCpuInfo const* placementCpuInfo, int rank = 0, int size = 1) = 0; @@ -107,7 +110,7 @@ class HostMemoryMoeWeightUpdater : public MoeWeightUpdaterBase private: static void copyWeights(MoeWeight const& src, MoeWeight const& dst, cudaStream_t stream); - static void copyWeightsCpu(MoeWeight const& src, MoeWeight const& dst, int rank, int size); + static void copyWeightsCpu(MoeWeight const& src, MoeWeight const& dst, int rank, int size, bool dstIsDeviceAddress); void finalizeHostWeight(); bool mHostWeightsFinalized = false; std::map> mHostWeights; @@ -129,7 +132,7 @@ class SingleLayerMoeLoadBalancer void addSingleHostWeight(int expertId, std::string const& name, MoeWeight hostWeight); // set initial weight assignments for each slot - // index is the global slot id, value is the expert id + // index is the global slot id, value is the expert id, or -1 for an empty slot // should bind to python void setInitialWeightAssignments(std::vector const& initialWeightAssignments); @@ -149,6 +152,11 @@ class SingleLayerMoeLoadBalancer return mCpuPlacementInfo.oldRankExpertIds; } + // should bind to python + // Return the local slots that the initial or the last installed placement leaves empty, which EPLB's updates leave + // empty too. Unlike getOldRankExpertIds, they can be read while an update runs. + std::vector getUnusableLocalSlots() const; + tensorrt_llm::kernels::MoeLoadBalanceMetaInfo getMetaInfo() const { return mMetaInfo; @@ -166,6 +174,11 @@ class SingleLayerMoeLoadBalancer return mGpuPlacementGpuAccess; } + // Return whether routing uses the direct expert-to-slot kernel, because every expert has a single replica. Under + // fault tolerance this is fixed by the initial placement: CUDA graphs capture the routing kernel, so every later + // placement must keep a single replica per expert (checkKeepsSingleReplicaPerExpert). + bool usesSingleReplicaRouting() const; + tensorrt_llm::kernels::MoeLoadBalanceSingleLayerSignal* getSignal() { return mSingleLayerSignal; @@ -191,11 +204,21 @@ class SingleLayerMoeLoadBalancer void waitCpuStage(); void maybeStartUpdateWeights(); void waitLastUpdateDone(); + void resetAfterAbandon(); void validateMaskOnly(std::vector const& deadRankMask) const; void reconfigureMaskOnly(std::vector const& deadRankMask); std::vector computeMaskOnlyReplicaCounts(std::vector const& deadRankMask) const; + std::vector computeReplicaCounts(std::vector const& deadRankMask, + std::vector> const& rankExpertIds, char const* operation) const; + void validatePlacement( + std::vector const& deadRankMask, std::vector> const& rankExpertIds) const; + void installPlacement(std::vector const& deadRankMask, std::vector> const& rankExpertIds); + void checkKeepsSingleReplicaPerExpert(std::vector const& expertReplicaCount) const; + void updateSingleReplicaPerExpert(); MoeLoadBalancer* mMoeLoadBalancer = nullptr; + // Whether the initial placement has a single replica per expert. + bool mSingleReplicaPerExpert = false; std::unique_ptr mWeightUpdater; @@ -210,6 +233,11 @@ class SingleLayerMoeLoadBalancer int64_t mLastUpdateTaskId = -1; + // Per global slot, 1 if a live rank's slot holds no expert (-1 in the initial or an installed placement), so EPLB's + // updates must leave it empty. Slots of dead ranks are excluded by the dead rank mask instead. Empty if no slot is + // unusable. + std::vector mUnusableSlots; + cudaEvent_t mUpdateWeightsDoneEvent = nullptr; tensorrt_llm::kernels::MoeLoadBalanceMetaInfo mMetaInfo; tensorrt_llm::kernels::MoeLoadBalanceStatisticInfo mStatisticInfo; @@ -265,7 +293,7 @@ class MultiThreadWorker class MoeLoadBalancer { public: - MoeLoadBalancer(int epRank, int epSize, int layerUpdatesPerIter); + MoeLoadBalancer(int epRank, int epSize, int layerUpdatesPerIter, bool faultTolerance = false); ~MoeLoadBalancer(); // Add a new layer to the load balancer @@ -291,6 +319,23 @@ class MoeLoadBalancer // The caller must separately gate degraded-mode capacity/HBM headroom before invoking it. void reconfigureMaskOnly(std::vector const& deadRanks); + // should bind to python + // Set the failed ranks and install a placement per layer, while no iteration runs. Every rank must finish restoring + // before any rank routes with the new placement. The caller loads every local slot whose expert the placement + // changes before the call. + void restorePlacement( + std::vector const& deadRanks, std::vector>> const& layerRankExpertIds); + + // should bind to python + // End the online EPLB work of the iterations started before a rank failure, and restart the statistics and the + // update plan so that every survivor updates the same layers. Every launched kernel must be able to finish. + void abandonIterations(); + + // should bind to python + // Check that no iteration runs or is queued and the online EPLB worker is done with the last one, so that the + // caller may write local slots. + void checkSlotsWritable(); + // Test interface to use GPU to do memcpy test functionality void setUseGpuMemcpy(bool useGpuMemcpy = false) { @@ -303,11 +348,23 @@ class MoeLoadBalancer void startThreads(); + void checkRecoveryCall(char const* action) const; + + // Like checkRecoveryCall, and also check that no iteration is queued and the online EPLB worker is done with the + // last one. Call with mWorkerThreadMutex held. + void checkUpdatesEnded(char const* action) const; + // worker thread is used to wait for gpu update done signal // and also used to trigger cpu update and wait for it done and signal gpu void workerThread(); std::mutex mWorkerThreadMutex; std::condition_variable mWorkerThreadCondition; + // Wide-EP fault tolerance: slots may hold no expert (-1), a layer with a single replica per expert uses the direct + // routing kernel, and the recovery calls (restorePlacement, abandonIterations, checkSlotsWritable) are allowed. + bool mFaultTolerance = false; + int64_t mWorkerDoneIterId = -1; + std::condition_variable mWorkerDoneCondition; + std::atomic mAbandoning{false}; bool mWorkerThreadStopped = true; int64_t mWarmUpUntilIter = -1; @@ -361,10 +418,13 @@ class MoeLoadBalancer }; // functions exposed for testing +// Slots that unusableSlotMask marks are left -1. void doReplication(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* const expertLoadFactor, - MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask = nullptr); + MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask = nullptr, + std::vector const* unusableSlotMask = nullptr); void doPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo, float* const expertLoadFactor, - MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask = nullptr); + MoePlacementCpuInfo* cpuPlacement, std::vector const* deadRankMask = nullptr, + std::vector const* unusableSlotMask = nullptr); } // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.cpp b/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.cpp new file mode 100644 index 000000000000..76baba9617d7 --- /dev/null +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.cpp @@ -0,0 +1,530 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "unbackedSlotMemory.h" +#include "hostAccessibleDeviceAllocator.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaDriverWrapper.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/runtime/memoryCounters.h" + +#include +#include +#include +#include + +namespace tensorrt_llm::runtime +{ + +namespace +{ + +CUmemAllocationProp slotMemoryProp(int device, bool hostAccessible = false) +{ + CUmemAllocationProp prop{}; + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + prop.location.id = device; + prop.allocFlags.gpuDirectRDMACapable = hostAccessible ? 1 : 0; + return prop; +} + +// Registers the GDRCopy host mapping of a backed chunk with HostAccessibleDeviceAllocator, so EPLB's CPU copies reach +// the chunk by its device address. +struct HostMappingConfigurator : CUDAVirtualMemoryChunk::Configurator +{ + HostMappingConfigurator(CUdeviceptr address, size_t size) + : mAddress(address) + , mSize(size) + { + } + + void setup(CUmemGenericAllocationHandle) override + { + HostAccessibleDeviceAllocator::getInstance().mapDeviceMemory(reinterpret_cast(mAddress), mSize); + } + + void teardown(CUmemGenericAllocationHandle, bool) override + { + HostAccessibleDeviceAllocator::getInstance().unmapDeviceMemory(reinterpret_cast(mAddress)); + } + + CUdeviceptr mAddress; + size_t mSize; +}; + +CUmemAccessDesc slotMemoryAccess(int device) +{ + CUmemAccessDesc desc{}; + desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + desc.location.id = device; + desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + return desc; +} + +// cuMemMap maps whole allocations only, so the one-chunk allocation is mapped at each chunk of the range. +struct RepeatedMappingConfigurator : CUDAVirtualMemoryChunk::Configurator +{ + RepeatedMappingConfigurator(CUdeviceptr address, size_t size, size_t chunkSize, CUmemAccessDesc const& desc) + : mAddress(address) + , mSize(size) + , mChunkSize(chunkSize) + , mDesc(desc) + { + } + + void setup(CUmemGenericAllocationHandle handle) override + { + // Unmap on failure, because teardown runs only after a successful setup. + for (size_t offset = 0; offset < mSize; offset += mChunkSize) + { + CUresult const result = cuMemMap(mAddress + offset, mChunkSize, 0, handle, 0); + if (result != CUDA_SUCCESS) + { + if (offset > 0) + { + cuMemUnmap(mAddress, offset); + } + TLLM_CU_CHECK(result); + } + } + CUresult const result = cuMemSetAccess(mAddress, mSize, &mDesc, 1); + if (result != CUDA_SUCCESS) + { + cuMemUnmap(mAddress, mSize); + TLLM_CU_CHECK(result); + } + } + + void teardown(CUmemGenericAllocationHandle, bool) override + { + TLLM_CU_CHECK_FREE_RESOURCE(cuMemUnmap(mAddress, mSize)); + } + + CUdeviceptr mAddress; + size_t mSize; + size_t mChunkSize; + CUmemAccessDesc mDesc; +}; + +// Every chunk maps the same allocation, which mOwner keeps alive. +struct SharedCreator : CUDAVirtualMemoryChunk::Creator +{ + SharedCreator(std::shared_ptr owner, CUmemGenericAllocationHandle handle) + : mOwner(std::move(owner)) + , mHandle(handle) + { + } + + CUmemGenericAllocationHandle create() override + { + return mHandle; + } + + void release(CUmemGenericAllocationHandle, bool) override {} + + std::shared_ptr mOwner; + CUmemGenericAllocationHandle mHandle; +}; + +} // namespace + +struct UnbackedSlotMemory::Scratch +{ + Scratch(CUmemAllocationProp const& prop, size_t bytes) + : size(bytes) + { + TLLM_CU_CHECK(cuMemCreate(&handle, size, &prop, 0)); + MemoryCounters::getInstance().allocate(MemoryType::kGPU, size); + } + + Scratch(Scratch const&) = delete; + Scratch& operator=(Scratch const&) = delete; + + ~Scratch() + { + if (cuMemRelease(handle) != CUDA_SUCCESS) + { + TLLM_LOG_ERROR("Releasing the scratch of unbacked slot memory failed"); + } + MemoryCounters::getInstance().deallocate(MemoryType::kGPU, size); + } + + CUmemGenericAllocationHandle handle{}; + size_t size; +}; + +UnbackedSlotMemory& UnbackedSlotMemory::getInstance() +{ + // Never destroyed: CUDA may be torn down before static destructors run. + static auto* instance = new UnbackedSlotMemory; + return *instance; +} + +CUDAVirtualMemoryChunk UnbackedSlotMemory::mapChunk( + CUdeviceptr address, size_t size, bool backed, bool hostAccessible) const +{ + CUDAVirtualMemoryChunk::CreatorPtr creator; + CUDAVirtualMemoryChunk::Configurators configurators; + if (backed) + { + creator = std::make_unique>(slotMemoryProp(mDevice, hostAccessible), size); + configurators.push_back(std::make_unique(address, size, slotMemoryAccess(mDevice))); + if (hostAccessible) + { + configurators.push_back(std::make_unique(address, size)); + } + } + else + { + std::shared_ptr scratch = mScratch.lock(); + if (!scratch) + { + scratch = std::make_shared(slotMemoryProp(mDevice), mGranularity); + mScratch = scratch; + } + creator = std::make_unique(scratch, scratch->handle); + configurators.push_back( + std::make_unique(address, size, mGranularity, slotMemoryAccess(mDevice))); + } + CUDAVirtualMemoryChunk chunk(std::move(creator), std::move(configurators)); + // If this throws, the chunk's destructor releases what it made. + chunk.materialize(); + return chunk; +} + +void UnbackedSlotMemory::releaseChunks(Allocation& allocation) noexcept +{ + auto releaseLogged = [](CUDAVirtualMemoryChunk& chunk) + { + if (chunk.status() != CUDAVirtualMemoryChunk::MATERIALIZED) + { + return; + } + try + { + chunk.release(); + } + catch (std::exception const& error) + { + TLLM_LOG_ERROR("Releasing unbacked slot memory failed: %s", error.what()); + } + }; + for (auto& piece : allocation.pieces) + { + releaseLogged(piece.chunk); + } + for (auto& chunk : allocation.alwaysBacked) + { + releaseLogged(chunk); + } +} + +UnbackedSlotMemory::Layout UnbackedSlotMemory::plan( + size_t slotBytes, int slotCount, std::vector const& unbackedSlots) +{ + TLLM_CHECK_WITH_INFO(slotBytes > 0 && slotCount > 0, "No slots to allocate"); + int device = -1; + TLLM_CUDA_CHECK(cudaGetDevice(&device)); + TLLM_CHECK_WITH_INFO(mDevice < 0 || mDevice == device, "Unbacked slot memory serves one device per process"); + if (mDevice < 0) + { + auto const prop = slotMemoryProp(device); + TLLM_CU_CHECK(cuMemGetAllocationGranularity(&mGranularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); + mDevice = device; + } + size_t const slots = static_cast(slotCount); + Layout layout{std::vector(slots, 0), common::divUp(slotBytes * slots, mGranularity), {}}; + for (int slot : unbackedSlots) + { + TLLM_CHECK_WITH_INFO(slot >= 0 && slot < slotCount, "Unbacked slot %d is not in [0, %d)", slot, slotCount); + layout.unbacked[slot] = 1; + } + layout.slotOf.resize(layout.chunks); + for (size_t chunk = 0; chunk < layout.chunks; ++chunk) + { + size_t const first = firstSlotOfChunk(chunk, slotBytes, slots); + size_t const last = lastSlotOfChunk(chunk, slotBytes, slots); + bool inPiece = true; + for (size_t slot = first; slot <= last && inPiece; ++slot) + { + inPiece = slot == slots || layout.unbacked[slot] != 0; + } + if (!inPiece) + { + layout.slotOf[chunk] = -1; + } + else if (first == last) + { + layout.slotOf[chunk] = static_cast(first); + } + else + { + layout.slotOf[chunk] = -2 - static_cast(chunk); + } + } + return layout; +} + +size_t UnbackedSlotMemory::firstSlotOfChunk(size_t chunk, size_t slotBytes, size_t slots) const +{ + return std::min(chunk * mGranularity / slotBytes, slots); +} + +size_t UnbackedSlotMemory::lastSlotOfChunk(size_t chunk, size_t slotBytes, size_t slots) const +{ + return std::min(((chunk + 1) * mGranularity - 1) / slotBytes, slots); +} + +bool UnbackedSlotMemory::savesMemory(size_t slotBytes, int slotCount, std::vector const& unbackedSlots) +{ + std::lock_guard lock(mMutex); + auto const slotOf = plan(slotBytes, slotCount, unbackedSlots).slotOf; + return std::any_of(slotOf.begin(), slotOf.end(), [](int64_t slot) { return slot != -1; }); +} + +void* UnbackedSlotMemory::allocate( + size_t slotBytes, int slotCount, std::vector const& unbackedSlots, bool hostAccessible) +{ + TLLM_CHECK_WITH_INFO(!hostAccessible || HostAccessibleDeviceAllocator::getInstance().usesGdrCopy(), + "Host accessible unbacked slot memory needs host accessible device memory to be GDRCopy mappings"); + std::lock_guard lock(mMutex); + auto const layout = plan(slotBytes, slotCount, unbackedSlots); + auto const& slotOf = layout.slotOf; + size_t const chunks = layout.chunks; + size_t const slots = static_cast(slotCount); + + CUdeviceptr base = 0; + TLLM_CU_CHECK(cuMemAddressReserve(&base, chunks * mGranularity, mGranularity, 0, 0)); + Allocation allocation{ + std::shared_ptr(reinterpret_cast(base), + [size = chunks * mGranularity](void* ptr) { cuMemAddressFree(reinterpret_cast(ptr), size); }), + layout.unbacked, std::vector(slots, 0), {}, {}, hostAccessible}; + try + { + for (size_t chunk = 0; chunk < chunks;) + { + size_t run = chunk; + while (run < chunks && slotOf[run] == slotOf[chunk]) + { + ++run; + } + CUdeviceptr const address = base + chunk * mGranularity; + size_t const size = (run - chunk) * mGranularity; + if (slotOf[chunk] == -1) + { + allocation.alwaysBacked.push_back(mapChunk(address, size, /*backed=*/true, hostAccessible)); + } + else + { + allocation.pieces.push_back( + Piece{address, size, static_cast(firstSlotOfChunk(chunk, slotBytes, slots)), + static_cast(lastSlotOfChunk(run - 1, slotBytes, slots)), /*backed=*/false, + mapChunk(address, size, /*backed=*/false, hostAccessible)}); + } + chunk = run; + } + } + catch (...) + { + releaseChunks(allocation); + throw; + } + mAllocations.emplace(base, std::move(allocation)); + return reinterpret_cast(base); +} + +void UnbackedSlotMemory::free(void* ptr) noexcept +{ + std::lock_guard lock(mMutex); + auto it = mAllocations.find(reinterpret_cast(ptr)); + if (it == mAllocations.end()) + { + TLLM_LOG_ERROR("%p is not unbacked slot memory", ptr); + return; + } + // Unlike cudaFree, cuMemUnmap does not wait for queued work, and work on any stream may use the + // range. A tensor deleter may run with another device current. + int current = -1; + TLLM_CUDA_CHECK_WARN(cudaGetDevice(¤t)); + if (current != mDevice) + { + TLLM_CUDA_CHECK_WARN(cudaSetDevice(mDevice)); + } + // The synchronization fails during a CUDA graph capture that forbids it. Queued or captured work may then still + // use the range, so the range stays mapped and leaks. + cudaError_t const syncError = cudaDeviceSynchronize(); + if (syncError == cudaSuccess) + { + releaseChunks(it->second); + mAllocations.erase(it); + } + else + { + TLLM_LOG_ERROR("Leaking unbacked slot memory %p because synchronizing device %d failed: %s", ptr, mDevice, + cudaGetErrorString(syncError)); + } + if (current != mDevice && current >= 0) + { + TLLM_CUDA_CHECK_WARN(cudaSetDevice(current)); + } +} + +UnbackedSlotMemory::Allocation& UnbackedSlotMemory::findAllocation(void* ptr, std::vector const& slots) +{ + auto it = mAllocations.find(reinterpret_cast(ptr)); + TLLM_CHECK_WITH_INFO(it != mAllocations.end(), "%p is not unbacked slot memory", ptr); + auto& allocation = it->second; + int const slotCount = static_cast(allocation.unbacked.size()); + for (int slot : slots) + { + TLLM_CHECK_WITH_INFO(slot >= 0 && slot < slotCount && allocation.unbacked[slot] != 0, + "Slot %d is not one of the unbacked slots of %d", slot, slotCount); + } + return allocation; +} + +bool UnbackedSlotMemory::anyBacked(Piece const& piece, std::vector const& backed) +{ + int const lastSlot = std::min(piece.lastSlot, static_cast(backed.size()) - 1); + return std::any_of(backed.begin() + piece.firstSlot, backed.begin() + lastSlot + 1, + [](uint8_t slotBacked) { return slotBacked != 0; }); +} + +size_t UnbackedSlotMemory::bytesToBack(void* ptr, std::vector const& slots) +{ + std::lock_guard lock(mMutex); + auto const& allocation = findAllocation(ptr, slots); + auto backed = allocation.backed; + for (int slot : slots) + { + backed[slot] = 1; + } + size_t bytes = 0; + for (auto const& piece : allocation.pieces) + { + bytes += !piece.backed && anyBacked(piece, backed) ? piece.size : 0; + } + return bytes; +} + +bool UnbackedSlotMemory::isUnbacked(void const* address) +{ + std::lock_guard lock(mMutex); + auto const target = reinterpret_cast(address); + auto it = mAllocations.upper_bound(target); + if (it == mAllocations.begin()) + { + return false; + } + --it; + return it->second.hostAccessible + && std::any_of(it->second.pieces.begin(), it->second.pieces.end(), + [target](Piece const& piece) + { return !piece.backed && target >= piece.address && target < piece.address + piece.size; }); +} + +size_t UnbackedSlotMemory::setBacked(std::vector const& ranges, bool backed) +{ + std::lock_guard lock(mMutex); + std::vector allocations; + allocations.reserve(ranges.size()); + for (auto const& range : ranges) + { + allocations.push_back(&findAllocation(range.ptr, range.slots)); + } + int device = -1; + TLLM_CUDA_CHECK(cudaGetDevice(&device)); + TLLM_CHECK_WITH_INFO( + device == mDevice, "Unbacked slot memory of device %d changed with device %d current", mDevice, device); + std::map> requested; + for (size_t i = 0; i < ranges.size(); ++i) + { + auto& slotsBacked = requested.try_emplace(allocations[i], allocations[i]->backed).first->second; + for (int slot : ranges[i].slots) + { + slotsBacked[slot] = static_cast(backed); + } + } + std::vector> changed; + for (auto& [allocation, slotsBacked] : requested) + { + for (auto& piece : allocation->pieces) + { + if (anyBacked(piece, slotsBacked) != piece.backed) + { + changed.emplace_back(&piece, allocation->hostAccessible); + } + } + } + size_t bytes = 0; + if (!changed.empty()) + { + // Unlike cudaFree, cuMemUnmap does not wait for queued work on any stream that uses the slots. + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + size_t remapped = 0; + try + { + for (; remapped < changed.size(); ++remapped) + { + remap(*changed[remapped].first, changed[remapped].second); + bytes += changed[remapped].first->size; + } + } + catch (...) + { + // Restore the pieces remapped before the failure, so the call changes nothing. + while (remapped-- > 0) + { + try + { + remap(*changed[remapped].first, changed[remapped].second); + } + catch (std::exception const& error) + { + TLLM_LOG_ERROR("Mapping unbacked slot memory back failed: %s", error.what()); + } + } + throw; + } + } + for (auto& [allocation, slotsBacked] : requested) + { + allocation->backed = std::move(slotsBacked); + } + return bytes; +} + +void UnbackedSlotMemory::remap(Piece& piece, bool hostAccessible) const +{ + // An address holds one mapping at a time. If the new mapping fails, the old state is mapped again. + piece.chunk.release(); + try + { + piece.chunk = mapChunk(piece.address, piece.size, !piece.backed, hostAccessible); + } + catch (...) + { + piece.chunk = mapChunk(piece.address, piece.size, piece.backed, hostAccessible); + throw; + } + piece.backed = !piece.backed; +} + +} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h b/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h new file mode 100644 index 000000000000..36d7e738da11 --- /dev/null +++ b/cpp/tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/runtime/virtualMemory.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace tensorrt_llm::runtime +{ + +//! \brief Device storage for an EPLB slot tensor whose unbacked (spare) slots hold no device memory of their own. +//! +//! The range stays fully mapped at a fixed address, because CUDA graphs capture it. It is mapped in chunks of the +//! allocation granularity. Chunks wholly inside unbacked slots map one shared scratch chunk, so a stray access reads +//! or writes scratch instead of faulting. Backing a slot maps memory of its own at the same address. +class UnbackedSlotMemory +{ +public: + static UnbackedSlotMemory& getInstance(); + + //! Return whether allocate() would leave at least one chunk unbacked on the current device. If it would not, every + //! byte gets memory of its own, so a plain allocation serves as well. + bool savesMemory(size_t slotBytes, int slotCount, std::vector const& unbackedSlots); + //! Allocate slotCount slots of slotBytes each on the current device, leaving unbackedSlots unbacked. + void* allocate(size_t slotBytes, int slotCount, std::vector const& unbackedSlots, bool hostAccessible = false); + //! Synchronize the range's device, then unmap and release the range. If the synchronization fails, as it does + //! during a CUDA graph capture, log an error and leak the range instead. + void free(void* ptr) noexcept; + + struct SlotRange + { + void* ptr; + std::vector slots; + }; + + //! Map memory of their own (contents undefined) or the scratch at the slots and return the bytes changed. + //! A changed mapping synchronizes the device, so no kernel may use the slots and no CUDA graph capture may run. + //! On failure every slot keeps its state. + size_t setBacked(std::vector const& ranges, bool backed); + + size_t setBacked(void* ptr, std::vector const& slots, bool backed) + { + return setBacked({SlotRange{ptr, slots}}, backed); + } + + //! Return the device bytes that setBacked(ptr, slots, true) would map now. + size_t bytesToBack(void* ptr, std::vector const& slots); + + //! Return whether address lies in an unbacked slot of host accessible slot memory. The slot maps the scratch, which + //! has no host mapping, until backing it maps memory of its own for the host. + bool isUnbacked(void const* address); + +private: + struct Scratch; + + //! A run of chunks whose bytes all belong to the unbacked slots [firstSlot, lastSlot], mapped as one unit. It maps + //! the scratch while none of its slots is backed, and memory of its own once any of them is. + struct Piece + { + CUdeviceptr address; + size_t size; + int firstSlot; + // The slot count if the piece ends in the padding after the last slot, which rounds the range up to the + // granularity. + int lastSlot; + bool backed; + CUDAVirtualMemoryChunk chunk; + }; + + struct Allocation + { + // Declared first so that it is freed after the chunks are unmapped. + std::shared_ptr reservation; + // Per slot, 1 if the slot was allocated unbacked. + std::vector unbacked; + // Per slot, 1 if setBacked currently backs the unbacked slot. + std::vector backed; + std::vector pieces; + // The chunks holding bytes of a slot allocated backed. They keep memory of their own for the allocation's + // lifetime. + std::vector alwaysBacked; + bool hostAccessible; + }; + + struct Layout + { + std::vector unbacked; + size_t chunks; + // Per chunk: the unbacked slot holding all its bytes; -1 if any byte belongs to a slot allocated backed (the + // chunk is always backed); or -2 - chunk if it spans several unbacked slots. A run of chunks with one value is + // a piece. A chunk spanning several slots gets a value of its own, so adjacent spanning chunks never merge and + // backing a slot backs only the chunks that hold its bytes. + std::vector slotOf; + }; + + UnbackedSlotMemory() = default; + + //! Plan the layout of the slots on the current device, which becomes the device this instance serves. + Layout plan(size_t slotBytes, int slotCount, std::vector const& unbackedSlots); + //! Return the first or last slot holding bytes of chunk, counting the tail as slot number slots. + size_t firstSlotOfChunk(size_t chunk, size_t slotBytes, size_t slots) const; + size_t lastSlotOfChunk(size_t chunk, size_t slotBytes, size_t slots) const; + + Allocation& findAllocation(void* ptr, std::vector const& slots); + //! Return whether backed marks any slot of piece, which is the state the piece must be mapped in. + static bool anyBacked(Piece const& piece, std::vector const& backed); + + //! Map [address, address + size) to memory of its own if backed, else to the scratch. + CUDAVirtualMemoryChunk mapChunk(CUdeviceptr address, size_t size, bool backed, bool hostAccessible) const; + //! Map a piece to its other state, or restore its state and throw. + void remap(Piece& piece, bool hostAccessible) const; + static void releaseChunks(Allocation& allocation) noexcept; + + std::mutex mMutex; + std::map mAllocations; + int mDevice = -1; + size_t mGranularity = 0; + // The chunks that map the scratch keep it alive. + mutable std::weak_ptr mScratch; +}; + +} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/thop/moe/loadBalance/moeLoadBalanceOp.cpp b/cpp/tensorrt_llm/thop/moe/loadBalance/moeLoadBalanceOp.cpp index e128bbcef424..e5c32e71fac7 100644 --- a/cpp/tensorrt_llm/thop/moe/loadBalance/moeLoadBalanceOp.cpp +++ b/cpp/tensorrt_llm/thop/moe/loadBalance/moeLoadBalanceOp.cpp @@ -162,7 +162,8 @@ torch::Tensor moeLoadBalanceRouting( auto tokenRoutedSlotIds = torch::empty_like(tokenSelectedExperts); tensorrt_llm::kernels::moeComputeRouteDevice(metaInfo, loadBalancer->getGpuPlacementInfo(), - tokenSelectedExperts.data_ptr(), tokenRoutedSlotIds.data_ptr(), tokenCount, offsetByEpRank, stream); + tokenSelectedExperts.data_ptr(), tokenRoutedSlotIds.data_ptr(), tokenCount, offsetByEpRank, stream, + loadBalancer->usesSingleReplicaRouting()); return tokenRoutedSlotIds; } diff --git a/cpp/tensorrt_llm/thop/moe/loadBalance/moeUnbackedSlotOp.cpp b/cpp/tensorrt_llm/thop/moe/loadBalance/moeUnbackedSlotOp.cpp new file mode 100644 index 000000000000..01e4aafe3298 --- /dev/null +++ b/cpp/tensorrt_llm/thop/moe/loadBalance/moeUnbackedSlotOp.cpp @@ -0,0 +1,122 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h" +#include "tensorrt_llm/thop/thUtils.h" + +#include +#include +#include +#include +#include +#include + +#include + +TRTLLM_NAMESPACE_BEGIN + +namespace torch_ext +{ + +// Return whether leaving unbackedSlots unbacked saves device memory on the current device. +bool moeUnbackedSlotsSaveMemory(int64_t slotBytes, int64_t slotCount, at::IntArrayRef unbackedSlots) +{ + TORCH_CHECK(slotBytes > 0 && slotCount > 0, "expects slots of at least one byte"); + return tensorrt_llm::runtime::UnbackedSlotMemory::getInstance().savesMemory(static_cast(slotBytes), + static_cast(slotCount), std::vector(unbackedSlots.begin(), unbackedSlots.end())); +} + +// Create a tensor of slots whose unbackedSlots map a shared scratch. Releasing its storage synchronizes the device, +// so the tensor must not be freed during CUDA graph capture. +at::Tensor moeUnbackedSlotTensor( + at::IntArrayRef sizes, at::ScalarType dtype, at::IntArrayRef unbackedSlots, bool hostAccessible) +{ + TORCH_CHECK(!sizes.empty() && sizes[0] > 0, "expects the sizes of a tensor of slots"); + size_t const byteSize = c10::multiply_integers(sizes) * c10::elementSize(dtype); + TORCH_CHECK(byteSize > 0, "expects slots of at least one byte"); + int const slotCount = static_cast(sizes[0]); + at::Device const device(at::kCUDA, c10::cuda::current_device()); + void* devPtr = tensorrt_llm::runtime::UnbackedSlotMemory::getInstance().allocate( + byteSize / slotCount, slotCount, std::vector(unbackedSlots.begin(), unbackedSlots.end()), hostAccessible); + c10::DataPtr dp( + devPtr, devPtr, [](void* ptr) { tensorrt_llm::runtime::UnbackedSlotMemory::getInstance().free(ptr); }, device); + auto storageImpl = c10::make_intrusive(c10::StorageImpl::use_byte_size_t(), byteSize, + std::move(dp), c10::GetAllocator(device.type()), /*resizable=*/false); + return at::empty({0}, at::TensorOptions().dtype(dtype).device(device)) + .set_(at::Storage(storageImpl), /*storage_offset=*/0, sizes); +} + +// Map device memory of their own at the slots' addresses, or the shared scratch again, and return the bytes changed. +// A changed mapping synchronizes the device, so no kernel may use the slots and no CUDA graph capture may run. +// slots concatenates the slot ids of every tensor, and slotsPerTensor[i] is how many of them belong to tensors[i]. +int64_t moeUnbackedSlotsSetBacked( + at::TensorList tensors, at::IntArrayRef slots, at::IntArrayRef slotsPerTensor, bool backed) +{ + TORCH_CHECK( + !tensors.empty() && tensors.size() == slotsPerTensor.size(), "expects a slot count per tensor of slots"); + std::vector ranges; + ranges.reserve(tensors.size()); + auto next = slots.begin(); + for (size_t i = 0; i < tensors.size(); ++i) + { + TORCH_CHECK(tensors[i].device() == tensors.front().device() && tensors[i].device().is_cuda(), + "expects CUDA tensors of slots on one device"); + TORCH_CHECK(slotsPerTensor[i] >= 0 && slotsPerTensor[i] <= slots.end() - next, + "expects slotsPerTensor to sum to the slots"); + ranges.push_back({tensors[i].storage().data_ptr().get(), std::vector(next, next + slotsPerTensor[i])}); + next += slotsPerTensor[i]; + } + TORCH_CHECK(next == slots.end(), "expects slotsPerTensor to sum to the slots"); + c10::cuda::CUDAGuard const deviceGuard(tensors.front().device()); + return static_cast(tensorrt_llm::runtime::UnbackedSlotMemory::getInstance().setBacked(ranges, backed)); +} + +// Return the device bytes that backing the slots would map now. +int64_t moeUnbackedSlotsBytesToBack(at::Tensor const& tensor, at::IntArrayRef slots) +{ + TORCH_CHECK(tensor.device().is_cuda(), "expects a CUDA tensor of slots"); + return static_cast(tensorrt_llm::runtime::UnbackedSlotMemory::getInstance().bytesToBack( + tensor.storage().data_ptr().get(), std::vector(slots.begin(), slots.end()))); +} + +} // namespace torch_ext + +TRTLLM_NAMESPACE_END + +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + m.def("moe_unbacked_slots_save_memory(int slot_bytes, int slot_count, int[] unbacked_slots) -> bool"); + m.def( + "moe_unbacked_slot_tensor(int[] sizes, ScalarType dtype, int[] unbacked_slots, bool host_accessible=False) -> " + "Tensor"); + m.def( + "moe_unbacked_slots_set_backed(Tensor(a!)[] tensors, int[] slots, int[] slots_per_tensor, bool backed) -> int"); + m.def("moe_unbacked_slots_bytes_to_back(Tensor tensor, int[] slots) -> int"); +} + +TORCH_LIBRARY_IMPL(trtllm, CUDA, m) +{ + m.impl("moe_unbacked_slots_set_backed", &tensorrt_llm::torch_ext::moeUnbackedSlotsSetBacked); + m.impl("moe_unbacked_slots_bytes_to_back", &tensorrt_llm::torch_ext::moeUnbackedSlotsBytesToBack); +} + +// No tensor argument picks the backend: the slot tensor is created on the current CUDA device. +TORCH_LIBRARY_IMPL(trtllm, CompositeExplicitAutograd, m) +{ + m.impl("moe_unbacked_slots_save_memory", &tensorrt_llm::torch_ext::moeUnbackedSlotsSaveMemory); + m.impl("moe_unbacked_slot_tensor", &tensorrt_llm::torch_ext::moeUnbackedSlotTensor); +} diff --git a/cpp/tests/unit_tests/kernels/moe/moeLoadBalanceKernelTest.cpp b/cpp/tests/unit_tests/kernels/moe/moeLoadBalanceKernelTest.cpp index ec8bcb7e7fc2..86e15912df8c 100644 --- a/cpp/tests/unit_tests/kernels/moe/moeLoadBalanceKernelTest.cpp +++ b/cpp/tests/unit_tests/kernels/moe/moeLoadBalanceKernelTest.cpp @@ -14,10 +14,12 @@ * limitations under the License. */ +#include #include #include #include #include +#include #include #include #include @@ -586,6 +588,60 @@ TEST_P(MoeLoadBalanceRouteKernelTest, TestInvalidExpertIds) } } +TEST_P(MoeLoadBalanceRouteKernelTest, TestSingleReplicaPerExpertWithSpareSlots) +{ + auto param = GetParam(); + int const slotCount = param.epSize * param.slotCountPerRank; + + // Each expert has one replica in a random slot. The other slots are spare and hold no expert. The start offsets + // point one expert further, which only the kernels for several replicas read, so the result shows which kernel ran. + std::vector slots(slotCount); + std::iota(slots.begin(), slots.end(), 0); + std::shuffle(slots.begin(), slots.end(), mRng); + std::fill(mHostGlobalSlotIdsInfo.begin(), mHostGlobalSlotIdsInfo.end(), slotCount); + for (int expertId = 0; expertId < param.expertCount; ++expertId) + { + mHostExpertReplicaCount[expertId] = 1; + mHostExpertReplicaStartOffset[expertId] = (expertId + 1) % param.expertCount; + mHostGlobalSlotIdsInfo[expertId] = slots[expertId]; + } + ASSERT_EQ(cudaMemcpy(mPlacementInfo.expertReplicaCount, mHostExpertReplicaCount.data(), + param.expertCount * sizeof(int), cudaMemcpyHostToDevice), + cudaSuccess); + ASSERT_EQ(cudaMemcpy(mPlacementInfo.expertReplicaStartOffset, mHostExpertReplicaStartOffset.data(), + param.expertCount * sizeof(int), cudaMemcpyHostToDevice), + cudaSuccess); + ASSERT_EQ(cudaMemcpy(mPlacementInfo.globalSlotIds, mHostGlobalSlotIdsInfo.data(), slotCount * sizeof(int), + cudaMemcpyHostToDevice), + cudaSuccess); + + // Every seventh token selects an invalid expert. + std::uniform_int_distribution dist(0, param.expertCount - 1); + for (int i = 0; i < mTotalTokens; ++i) + { + mHostTokenSelectedExperts[i] = i % 7 == 0 ? -1 : dist(mRng); + } + ASSERT_EQ(cudaMemcpy(mDeviceTokenSelectedExperts, mHostTokenSelectedExperts.data(), mTotalTokens * sizeof(int), + cudaMemcpyHostToDevice), + cudaSuccess); + + moeComputeRouteDevice(mMetaInfo, mPlacementInfo, mDeviceTokenSelectedExperts, mDeviceTokenRoutedRankIds, + param.maxTokenCountPerRank, false, mStream, /*singleReplicaPerExpert=*/true); + ASSERT_EQ(cudaStreamSynchronize(mStream), cudaSuccess); + + ASSERT_EQ(cudaMemcpy(mHostTokenRoutedRankIds.data(), mDeviceTokenRoutedRankIds, mTotalTokens * sizeof(int), + cudaMemcpyDeviceToHost), + cudaSuccess); + + for (int i = 0; i < mTotalTokens; ++i) + { + int const expertId = mHostTokenSelectedExperts[i]; + int const expectedSlotId = expertId < 0 ? slotCount : mHostGlobalSlotIdsInfo[expertId]; + EXPECT_EQ(mHostTokenRoutedRankIds[i], expectedSlotId) + << "Token " << i << " routed to wrong slot. Expert ID: " << expertId; + } +} + INSTANTIATE_TEST_SUITE_P(MoeLoadBalanceRouteKernelTests, MoeLoadBalanceRouteKernelTest, ::testing::Values( // basic test scenarios diff --git a/cpp/tests/unit_tests/runtime/CMakeLists.txt b/cpp/tests/unit_tests/runtime/CMakeLists.txt index ca9485ad1e4f..8ff1bce56ed0 100644 --- a/cpp/tests/unit_tests/runtime/CMakeLists.txt +++ b/cpp/tests/unit_tests/runtime/CMakeLists.txt @@ -111,8 +111,10 @@ add_tllm_gtest( SOURCES moeLoadBalancerTest.cpp ../../../tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.cpp + ../../../tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.cpp LINK_LIBRARIES tllm::kernel_moe_load_balance + tllm::runtime_buffers tllm::runtime_host_accessible_memory LABELS "unit;runtime;gpu") diff --git a/cpp/tests/unit_tests/runtime/hostAccessibleDeviceAllocatorTest.cu b/cpp/tests/unit_tests/runtime/hostAccessibleDeviceAllocatorTest.cu index 60c1c8757b8d..1a646840f14c 100644 --- a/cpp/tests/unit_tests/runtime/hostAccessibleDeviceAllocatorTest.cu +++ b/cpp/tests/unit_tests/runtime/hostAccessibleDeviceAllocatorTest.cu @@ -16,10 +16,15 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmException.h" #include "tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h" +#include +#include +#include #include #include #include +#include #include namespace tensorrt_llm::runtime::unit_tests @@ -51,6 +56,42 @@ __global__ void writeDataOnDevice(int* devPtr, int size) } } +namespace +{ + +//! Set an environment variable for the scope of a test and restore its previous value after. +class ScopedEnv +{ +public: + ScopedEnv(char const* name, char const* value) + : mName(name) + { + char const* previous = std::getenv(name); + mHadPrevious = previous != nullptr; + mPrevious = mHadPrevious ? previous : ""; + setenv(name, value, 1); + } + + ~ScopedEnv() + { + if (mHadPrevious) + { + setenv(mName.c_str(), mPrevious.c_str(), 1); + } + else + { + unsetenv(mName.c_str()); + } + } + +private: + std::string mName; + bool mHadPrevious; + std::string mPrevious; +}; + +} // namespace + class HostAccessibleDeviceAllocatorTest : public ::testing::Test { protected: @@ -72,6 +113,15 @@ protected: allocator->DecRefCount(); } + // Initialize the allocator again with GDRCopy forced, so a test runs on GDRCopy wherever gdrdrv is loaded, even + // where GPU memory is a NUMA node. Test bodies are subclasses, which the allocator's friendship does not reach. + void reopenWithGdrCopyForced() + { + allocator->DecRefCount(); + ScopedEnv const forceGdrcopy("TRTLLM_EPLB_FORCE_GDRCOPY", "1"); + allocator->IncRefCount(); + } + HostAccessibleDeviceAllocator* allocator; }; @@ -264,7 +314,7 @@ TEST_F(HostAccessibleDeviceAllocatorTest, ManagedMemoryFallbackWithoutGdrCopy) void* hostPtr = allocator->getHostPtr(devPtr); ASSERT_NE(hostPtr, nullptr); - // 1. Copy from the host with memcpyToDevice, as the online EPLB weight updates do, then verify on the device. + // 1. Copy from the host with memcpyToDevice, then verify on the device. std::vector pattern(numInts); std::iota(pattern.begin(), pattern.end(), 0); allocator->memcpyToDevice(hostPtr, pattern.data(), allocSize); @@ -289,4 +339,90 @@ TEST_F(HostAccessibleDeviceAllocatorTest, ManagedMemoryFallbackWithoutGdrCopy) allocator->free(devPtr); } +// With fault tolerance, the online EPLB weight updates copy each row at its device address. Check that this writes the +// bytes that a copy through the row's host pointer writes, for a row inside the allocation. +TEST_F(HostAccessibleDeviceAllocatorTest, MemcpyToDeviceAddressMatchesMemcpyToDevice) +{ + if (!allocator->isSupported()) + { + GTEST_SKIP() << "Host accessible memory is not supported on this system."; + } + + constexpr size_t numInts = 256; + constexpr size_t allocSize = numInts * sizeof(int); + constexpr size_t rowOffset = 64; + constexpr size_t rowInts = 96; + std::vector pattern(rowInts); + std::iota(pattern.begin(), pattern.end(), 7); + + char* byHostPtr = static_cast(allocator->allocate(allocSize)); + char* byDeviceAddress = static_cast(allocator->allocate(allocSize)); + ASSERT_NE(byHostPtr, nullptr); + ASSERT_NE(byDeviceAddress, nullptr); + ASSERT_NE(allocator->getHostPtr(byHostPtr), nullptr); + ASSERT_NE(allocator->getHostPtr(byDeviceAddress), nullptr); + TLLM_CUDA_CHECK(cudaMemset(byHostPtr, 0, allocSize)); + TLLM_CUDA_CHECK(cudaMemset(byDeviceAddress, 0, allocSize)); + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + + allocator->memcpyToDevice(static_cast(allocator->getHostPtr(byHostPtr)) + rowOffset * sizeof(int), + pattern.data(), rowInts * sizeof(int)); + allocator->memcpyToDeviceAddress(byDeviceAddress + rowOffset * sizeof(int), pattern.data(), rowInts * sizeof(int)); + + std::vector expected(numInts); + std::vector actual(numInts); + TLLM_CUDA_CHECK(cudaMemcpy(expected.data(), byHostPtr, allocSize, cudaMemcpyDeviceToHost)); + TLLM_CUDA_CHECK(cudaMemcpy(actual.data(), byDeviceAddress, allocSize, cudaMemcpyDeviceToHost)); + EXPECT_EQ(actual, expected); + EXPECT_EQ(actual[rowOffset], 7); + EXPECT_EQ(actual[rowOffset - 1], 0); + + allocator->free(byHostPtr); + allocator->free(byDeviceAddress); +} + +// With GDRCopy, a copy at a device address splits across adjacent host mappings, as the chunks of a backed spare slot +// are mapped one by one, and is refused outside every mapping. +TEST_F(HostAccessibleDeviceAllocatorTest, MemcpyToDeviceAddressSplitsAcrossMappings) +{ + reopenWithGdrCopyForced(); + if (!allocator->isSupported() || !allocator->usesGdrCopy()) + { + GTEST_SKIP() << "Device memory is not mapped for the host with GDRCopy on this system."; + } + + constexpr size_t kPageSize = 1UL << 16; + void* raw = nullptr; + TLLM_CUDA_CHECK(cudaMalloc(&raw, 4 * kPageSize)); + auto* const base = reinterpret_cast((reinterpret_cast(raw) + kPageSize - 1) & ~(kPageSize - 1)); + TLLM_CUDA_CHECK(cudaMemset(base, 0, 3 * kPageSize)); + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + allocator->mapDeviceMemory(base, kPageSize); + allocator->mapDeviceMemory(base + kPageSize, kPageSize); + ASSERT_NE(allocator->getHostPtr(base), nullptr); + ASSERT_NE(allocator->getHostPtr(base + kPageSize), nullptr); + + // A row that ends 256 bytes into the second mapping. + constexpr size_t kRowBytes = 1024; + char* const row = base + kPageSize - (kRowBytes - 256); + std::vector pattern(kRowBytes); + std::iota(pattern.begin(), pattern.end(), 1); + allocator->memcpyToDeviceAddress(row, pattern.data(), kRowBytes); + std::vector written(kRowBytes + 2); + TLLM_CUDA_CHECK(cudaMemcpy(written.data(), row - 1, written.size(), cudaMemcpyDeviceToHost)); + EXPECT_EQ(written.front(), 0); + EXPECT_EQ(written.back(), 0); + EXPECT_TRUE(std::equal(pattern.begin(), pattern.end(), written.begin() + 1)); + + // The third page is not mapped for the host. + EXPECT_THROW(allocator->memcpyToDeviceAddress(base + 2 * kPageSize, pattern.data(), kRowBytes), + tensorrt_llm::common::TllmException); + EXPECT_THROW(allocator->memcpyToDeviceAddress(base + 2 * kPageSize - 256, pattern.data(), kRowBytes), + tensorrt_llm::common::TllmException); + + allocator->unmapDeviceMemory(base); + allocator->unmapDeviceMemory(base + kPageSize); + TLLM_CUDA_CHECK(cudaFree(raw)); +} + } // namespace tensorrt_llm::runtime::unit_tests diff --git a/cpp/tests/unit_tests/runtime/moeLoadBalancerTest.cpp b/cpp/tests/unit_tests/runtime/moeLoadBalancerTest.cpp index 2bb1fe6dd5a3..b9520942ede9 100644 --- a/cpp/tests/unit_tests/runtime/moeLoadBalancerTest.cpp +++ b/cpp/tests/unit_tests/runtime/moeLoadBalancerTest.cpp @@ -14,12 +14,28 @@ * limitations under the License. */ +#include #include +#include "tensorrt_llm/common/cudaDriverWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/kernels/moe/loadBalance/moeLoadBalanceKernels.h" +#include "tensorrt_llm/runtime/moeLoadBalancer/hostAccessibleDeviceAllocator.h" #include "tensorrt_llm/runtime/moeLoadBalancer/moeLoadBalancer.h" +#include "tensorrt_llm/runtime/moeLoadBalancer/unbackedSlotMemory.h" +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include using namespace tensorrt_llm::runtime; @@ -369,6 +385,25 @@ TEST(MoeLoadBalancerMaskOnlyTest, DynamicPlacementRejectsMismatchedDeadRankMask) tensorrt_llm::common::TllmException); } +TEST(MoeLoadBalancerMaskOnlyTest, DynamicPlacementRejectsFewerActiveSlotsThanExperts) +{ + constexpr int kExpertCount = 4; + constexpr int kTopK = 2; + constexpr int kEpRank = 0; + constexpr int kEpSize = 4; + constexpr int kSlotCountPerRank = 1; + + tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo{kExpertCount, kTopK, kEpRank, kEpSize, kSlotCountPerRank}; + std::vector expertLoadFactor{1.0F, 1.0F, 1.0F, 1.0F}; + // Masking rank 2 leaves three slots for four experts. + std::vector deadRankMask{0, 0, 1, 0}; + + MoePlacementCpuInfo cpuPlacement; + EXPECT_THAT([&]() { doReplication(metaInfo, expertLoadFactor.data(), &cpuPlacement, &deadRankMask); }, + testing::Throws(testing::Property(&std::runtime_error::what, + testing::HasSubstr("Mask-only EPLB reconfigure would leave fewer active slots (3) than experts (4)")))); +} + TEST(MoeLoadBalancerMaskOnlyTest, DynamicPlacementRejectsReplicaCountMismatch) { constexpr int kExpertCount = 4; @@ -393,6 +428,84 @@ TEST(MoeLoadBalancerMaskOnlyTest, DynamicPlacementRejectsReplicaCountMismatch) tensorrt_llm::common::TllmException); } +namespace +{ + +// doReplication then doPlacement on a fresh placement of metaInfo's shape. +MoePlacementCpuInfo planPlacement(tensorrt_llm::kernels::MoeLoadBalanceMetaInfo const& metaInfo, + std::vector expertLoadFactor, std::vector const* deadRankMask, + std::vector const* unusableSlotMask) +{ + MoePlacementCpuInfo cpuPlacement; + cpuPlacement.rankExpertIds.assign(metaInfo.epSize, std::vector(metaInfo.slotCountPerRank, 99)); + doReplication(metaInfo, expertLoadFactor.data(), &cpuPlacement, deadRankMask, unusableSlotMask); + doPlacement(metaInfo, expertLoadFactor.data(), &cpuPlacement, deadRankMask, unusableSlotMask); + return cpuPlacement; +} + +} // namespace + +TEST(MoeLoadBalancerUnusableSlotTest, PlacementNeverFillsAnUnusableSlot) +{ + constexpr int kExpertCount = 4; + constexpr int kEpSize = 4; + constexpr int kSlotCountPerRank = 3; + constexpr int kSpareSlot = 2; + constexpr int kFailedRank = 2; + tensorrt_llm::kernels::MoeLoadBalanceMetaInfo metaInfo{ + kExpertCount, /*topK=*/2, /*epRank=*/0, kEpSize, kSlotCountPerRank}; + // The last slot of every rank is unusable, except on rank 1 (its spare slot now holds an expert, so it is usable). + std::vector unusableSlotMask(kEpSize * kSlotCountPerRank, 0); + for (int rank = 0; rank < kEpSize; ++rank) + { + unusableSlotMask[rank * kSlotCountPerRank + kSpareSlot] = static_cast(rank != 1); + } + std::vector const deadRankMask{0, 0, 1, 0}; + + for (auto const* failedRankMask : {static_cast const*>(nullptr), &deadRankMask}) + { + for (auto const& load : {std::vector{8.0F, 4.0F, 2.0F, 1.0F}, std::vector(kExpertCount, 0.0F)}) + { + auto const placement = planPlacement(metaInfo, load, failedRankMask, &unusableSlotMask); + int const usableSlotCount = failedRankMask == nullptr ? 9 : 7; + int replicaSum = 0; + for (int replicaCount : placement.expertReplicaCount) + { + EXPECT_GE(replicaCount, 1); + replicaSum += replicaCount; + } + EXPECT_EQ(replicaSum, usableSlotCount); + std::vector placedReplicas(kExpertCount, 0); + for (int rank = 0; rank < kEpSize; ++rank) + { + for (int slot = 0; slot < kSlotCountPerRank; ++slot) + { + int const expertId = placement.rankExpertIds[rank][slot]; + bool const usable = !(failedRankMask != nullptr && rank == kFailedRank) + && unusableSlotMask[rank * kSlotCountPerRank + slot] == 0; + if (!usable) + { + EXPECT_EQ(expertId, -1) << "rank " << rank << " slot " << slot; + continue; + } + ASSERT_GE(expertId, 0) << "rank " << rank << " slot " << slot; + ASSERT_LT(expertId, kExpertCount); + ++placedReplicas[expertId]; + } + } + EXPECT_EQ(placedReplicas, placement.expertReplicaCount); + } + } + + // Fewer usable slots than experts cannot be planned. + std::vector const tooFew(kEpSize * kSlotCountPerRank, 1); + EXPECT_THROW( + planPlacement(metaInfo, {1.0F, 1.0F, 1.0F, 1.0F}, nullptr, &tooFew), tensorrt_llm::common::TllmException); + std::vector const wrongSize(kEpSize, 0); + EXPECT_THROW( + planPlacement(metaInfo, {1.0F, 1.0F, 1.0F, 1.0F}, nullptr, &wrongSize), tensorrt_llm::common::TllmException); +} + TEST(MoeLoadBalancerMaskOnlyTest, ReconfigureMaskOnlyRemovesDeadRankSlots) { setenv("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1", 1); @@ -490,12 +603,626 @@ TEST(MoeLoadBalancerMaskOnlyTest, ReconfigureMaskOnlyFailsClosedForLastReplica) layer->setInitialWeightAssignments(initialAssignments); loadBalancer.finalizeModel(); - EXPECT_THROW(loadBalancer.reconfigureMaskOnly({kDeadRank}), tensorrt_llm::common::TllmException); + EXPECT_THAT([&]() { loadBalancer.reconfigureMaskOnly({kDeadRank}); }, + testing::Throws(testing::Property(&std::runtime_error::what, + testing::HasSubstr("Mask-only EPLB reconfigure would leave expert 2 with no surviving replica")))); EXPECT_EQ(layer->getPlacementCpuInfo()->oldRankExpertIds[kDeadRank][0], 2); loadBalancer.shutdown(); } +// The weight updater copies the slots of the layer's meta info EP rank, so a layer must carry its load balancer's rank. +TEST(MoeLoadBalancerLayerTest, LayerMetaInfoKeepsEpRank) +{ + setenv("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1", 1); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + + MoeLoadBalancer loadBalancer(/*epRank=*/1, /*epSize=*/2, /*layerUpdatesPerIter=*/0); + auto layer = loadBalancer.AddLayer(/*expertCount=*/2, /*topK=*/1, /*slotCountPerRank=*/1); + + EXPECT_EQ(layer->getMetaInfo().epRank, 1); + EXPECT_EQ(layer->getMetaInfo().epSize, 2); + + loadBalancer.shutdown(); +} + +namespace +{ + +//! Set an environment variable for the scope of a test and restore its previous value after. +class ScopedEnv +{ +public: + ScopedEnv(char const* name, char const* value) + : mName(name) + { + char const* previous = std::getenv(name); + mHadPrevious = previous != nullptr; + mPrevious = mHadPrevious ? previous : ""; + setenv(name, value, 1); + } + + ~ScopedEnv() + { + if (mHadPrevious) + { + setenv(mName.c_str(), mPrevious.c_str(), 1); + } + else + { + unsetenv(mName.c_str()); + } + } + +private: + std::string mName; + bool mHadPrevious; + std::string mPrevious; +}; + +// The GPU's part of a layer in a forward: takes the layer once the worker hands it over for iterId and, with handBack, +// records loads as the layer's statistics and hands the layer back. +void runLayerOnGpu(SingleLayerMoeLoadBalancer& layer, int64_t iterId, std::vector const& loads, bool handBack) +{ + auto* signal = layer.getSignal(); + int statisticEnabled = 0; + moeWaitSignalForGpuStageForTest(signal, &statisticEnabled); + int64_t const gpuIterId = signal->stepAndOwner >> 2; + EXPECT_EQ(gpuIterId, iterId); + if (handBack) + { + std::copy(loads.begin(), loads.end(), layer.getStatisticInfo()->expertLoadFactor); + moeSetSignalForCpuStageForTest(signal); + } +} + +bool cpuHoldsSignal(SingleLayerMoeLoadBalancer& layer) +{ + return (layer.getSignal()->stepAndOwner & tensorrt_llm::kernels::MoeLoadBalanceSingleLayerSignal::kDevice) + == tensorrt_llm::kernels::MoeLoadBalanceSingleLayerSignal::kCPU; +} + +// One EP rank of a model with online EPLB under Wide-EP fault tolerance, its weights in GPU-copied slots. Every byte of +// expert e's weight in layer l is expertByte(l, e); an empty slot's bytes are 0. +class SimulatedEpRank +{ +public: + static constexpr size_t kWeightBytes = 256; + + static uint8_t expertByte(int layerId, int expertId) + { + return static_cast(16 * (layerId + 1) + expertId + 1); + } + + SimulatedEpRank(int epRank, int epSize, int expertCount, int slotCountPerRank, int layerCount, + std::vector const& initialAssignments) + : mBalancer(epRank, epSize, /*layerUpdatesPerIter=*/1, /*faultTolerance=*/true) + , mEpRank(epRank) + { + mBalancer.setUseGpuMemcpy(true); + for (int layerId = 0; layerId < layerCount; ++layerId) + { + auto layer = mBalancer.AddLayer(expertCount, /*topK=*/1, slotCountPerRank); + for (int expertId = 0; expertId < expertCount; ++expertId) + { + void* hostWeight = nullptr; + TLLM_CUDA_CHECK(cudaMallocHost(&hostWeight, kWeightBytes)); + std::fill_n(static_cast(hostWeight), kWeightBytes, expertByte(layerId, expertId)); + mHostWeights.push_back(hostWeight); + layer->addSingleHostWeight(expertId, "weight", MoeWeight{hostWeight, 1, kWeightBytes, kWeightBytes}); + } + std::vector slots; + for (int slotId = 0; slotId < slotCountPerRank; ++slotId) + { + void* slot = nullptr; + TLLM_CUDA_CHECK(cudaMalloc(&slot, kWeightBytes)); + int const expertId = initialAssignments[epRank * slotCountPerRank + slotId]; + TLLM_CUDA_CHECK(cudaMemset(slot, expertId < 0 ? 0 : expertByte(layerId, expertId), kWeightBytes)); + slots.push_back(slot); + layer->addSingleWeightSlot(slotId, "weight", MoeWeight{slot, 1, kWeightBytes, kWeightBytes}); + } + layer->setInitialWeightAssignments(initialAssignments); + mSlots.push_back(slots); + mLayers.push_back(layer); + } + mBalancer.finalizeModel(); + } + + ~SimulatedEpRank() + { + mBalancer.shutdown(); + for (auto* hostWeight : mHostWeights) + { + EXPECT_EQ(cudaFreeHost(hostWeight), cudaSuccess); + } + for (auto const& slots : mSlots) + { + for (auto* slot : slots) + { + EXPECT_EQ(cudaFree(slot), cudaSuccess); + } + } + } + + SimulatedEpRank(SimulatedEpRank const&) = delete; + SimulatedEpRank& operator=(SimulatedEpRank const&) = delete; + + MoeLoadBalancer& balancer() + { + return mBalancer; + } + + SingleLayerMoeLoadBalancer& layer(int layerId) + { + return *mLayers[layerId]; + } + + // Load every local slot that placement fills with its expert's weight, as the caller of a restore does. + void loadSlots(int layerId, std::vector> const& placement) + { + for (size_t slotId = 0; slotId < mSlots[layerId].size(); ++slotId) + { + int const expertId = placement[mEpRank][slotId]; + if (expertId >= 0) + { + TLLM_CUDA_CHECK(cudaMemset(mSlots[layerId][slotId], expertByte(layerId, expertId), kWeightBytes)); + } + } + } + + // The first byte of a local slot. + uint8_t slotByte(int layerId, int slotId) + { + uint8_t byte = 0; + TLLM_CUDA_CHECK(cudaMemcpy(&byte, mSlots[layerId][slotId], 1, cudaMemcpyDeviceToHost)); + return byte; + } + + // Every local slot the layer's placement fills holds its expert's weight. + void expectSlotsHoldPlacement(int layerId) + { + auto const placement = mLayers[layerId]->getOldRankExpertIds(); + for (size_t slotId = 0; slotId < mSlots[layerId].size(); ++slotId) + { + int const expertId = placement[mEpRank][slotId]; + if (expertId < 0) + { + continue; + } + std::vector bytes(kWeightBytes); + TLLM_CUDA_CHECK(cudaMemcpy(bytes.data(), mSlots[layerId][slotId], kWeightBytes, cudaMemcpyDeviceToHost)); + EXPECT_EQ(static_cast(std::count(bytes.begin(), bytes.end(), expertByte(layerId, expertId))), + kWeightBytes) + << "rank " << mEpRank << ", layer " << layerId << ", slot " << slotId << " should hold expert " + << expertId; + } + } + + // The device table routing reads: per replica, its global slot. + std::vector routingTable(int layerId) + { + auto const metaInfo = mLayers[layerId]->getMetaInfo(); + std::vector slots(metaInfo.epSize * metaInfo.slotCountPerRank); + TLLM_CUDA_CHECK(cudaMemcpy(slots.data(), mLayers[layerId]->getGpuPlacementInfo().globalSlotIds, + slots.size() * sizeof(int), cudaMemcpyDeviceToHost)); + return slots; + } + +private: + MoeLoadBalancer mBalancer; + int mEpRank; + std::vector> mLayers; + std::vector mHostWeights; + std::vector> mSlots; +}; + +} // namespace + +TEST(MoeLoadBalancerAbandonTest, SurvivorsInterruptedAtDifferentLayersResumeFromOneRestoredPlacement) +{ + ScopedEnv const allowManagedFallback("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + using Placement = std::vector>; + constexpr int kEpSize = 4; + constexpr int kExpertCount = 5; + constexpr int kSlotCountPerRank = 3; + constexpr int kLayerCount = 2; + constexpr int kFailedRank = 3; + // Slot 2 of every rank is spare; only rank 3 holds expert 4. + Placement const healthy{{0, 1, -1}, {2, 0, -1}, {3, 1, -1}, {4, 2, -1}}; + std::vector initialAssignments; + for (auto const& rankExpertIds : healthy) + { + initialAssignments.insert(initialAssignments.end(), rankExpertIds.begin(), rankExpertIds.end()); + } + // Statistics that make EPLB's update replicate expert 4. + std::vector const loads{1, 1, 1, 1, 20}; + std::vector> survivors; + for (int epRank = 0; epRank < kFailedRank; ++epRank) + { + survivors.push_back(std::make_unique( + epRank, kEpSize, kExpertCount, kSlotCountPerRank, kLayerCount, initialAssignments)); + } + + // Iteration 0 updates layer 0 (the update plan: layer 0, layer 1, none). Rank 3 fails during it: rank 0's forward + // signals both layers, rank 1's only layer 0, rank 2's none (it stops inside layer 0). Rank 0 has also started + // iteration 1, whose forward never ran. + std::array const layersSignalled{2, 1, 0}; + for (int epRank = 0; epRank < kFailedRank; ++epRank) + { + auto& rank = *survivors[epRank]; + rank.balancer().startIter(0, /*enableStatistic=*/true, /*enableUpdateWeights=*/true); + for (int layerId = 0; layerId < layersSignalled[epRank]; ++layerId) + { + runLayerOnGpu(rank.layer(layerId), 0, loads, /*handBack=*/true); + } + if (layersSignalled[epRank] < kLayerCount) + { + runLayerOnGpu(rank.layer(layersSignalled[epRank]), 0, loads, /*handBack=*/false); + } + rank.balancer().endIter(0); + } + survivors[0]->balancer().startIter(1, /*enableStatistic=*/true, /*enableUpdateWeights=*/true); + survivors[0]->balancer().endIter(1); + // Rank 2's worker waits for layer 0, which its GPU never hands back: no placement can be restored yet. + EXPECT_THROW(survivors[2]->balancer().restorePlacement({kFailedRank}, {healthy, healthy}), + tensorrt_llm::common::TllmException); + + // Every survivor abandons: no statistics, every signal with the CPU. + for (auto& rank : survivors) + { + rank->balancer().abandonIterations(); + for (int layerId = 0; layerId < kLayerCount; ++layerId) + { + auto& layer = rank->layer(layerId); + auto const* loadFactor = layer.getStatisticInfo()->expertLoadFactor; + EXPECT_TRUE(std::all_of(loadFactor, loadFactor + kExpertCount, [](float load) { return load == 0.0F; })); + EXPECT_TRUE(cpuHoldsSignal(layer)); + } + } + // Ranks 0 and 1 started layer 0's update, rank 2 did not: the placements differ. + EXPECT_NE(survivors[0]->layer(0).getOldRankExpertIds(), healthy); + EXPECT_EQ(survivors[2]->layer(0).getOldRankExpertIds(), healthy); + + // One agreed placement: rank 3's expert 4 moves into rank 0's spare slot. Each survivor loads its local slots + // before the restore installs the placement. + Placement const restored{{0, 1, 4}, {2, 0, -1}, {3, 1, -1}, {4, 2, -1}}; + Placement const restoredActive{{0, 1, 4}, {2, 0, -1}, {3, 1, -1}, {-1, -1, -1}}; + for (auto& rank : survivors) + { + for (int layerId = 0; layerId < kLayerCount; ++layerId) + { + rank->loadSlots(layerId, restored); + } + rank->balancer().restorePlacement({kFailedRank}, {restored, restored}); + } + for (auto& rank : survivors) + { + for (int layerId = 0; layerId < kLayerCount; ++layerId) + { + EXPECT_EQ(rank->layer(layerId).getOldRankExpertIds(), restoredActive); + EXPECT_EQ(rank->routingTable(layerId), survivors[0]->routingTable(layerId)); + rank->expectSlotsHoldPlacement(layerId); + } + } + + // The next iteration completes everywhere: the update plan restarted, so every survivor updates layer 0 alike and + // leaves layer 1. + for (auto& rank : survivors) + { + int64_t const iterId = rank.get() == survivors[0].get() ? 2 : 1; + rank->balancer().startIter(iterId, /*enableStatistic=*/true, /*enableUpdateWeights=*/true); + for (int layerId = 0; layerId < kLayerCount; ++layerId) + { + runLayerOnGpu(rank->layer(layerId), iterId, loads, /*handBack=*/true); + } + rank->balancer().endIter(iterId); + // Waits for the weight update started. + rank->balancer().shutdown(); + } + auto const updated = survivors[0]->layer(0).getOldRankExpertIds(); + EXPECT_NE(updated, restoredActive); + EXPECT_EQ(updated[kFailedRank], (std::vector{-1, -1, -1})); + EXPECT_EQ(updated[1][2], -1); + EXPECT_EQ(updated[2][2], -1); + for (auto& rank : survivors) + { + EXPECT_EQ(rank->layer(0).getOldRankExpertIds(), updated); + EXPECT_EQ(rank->layer(1).getOldRankExpertIds(), restoredActive); + rank->expectSlotsHoldPlacement(0); + rank->expectSlotsHoldPlacement(1); + } + // The update filled every slot that the restore filled, so the unusable slots are the restore's empty spare slots + // in both layers. + for (int epRank = 0; epRank < kFailedRank; ++epRank) + { + std::vector const emptySpareSlots = epRank == 0 ? std::vector{} : std::vector{2}; + EXPECT_EQ(survivors[epRank]->layer(0).getUnusableLocalSlots(), emptySpareSlots); + EXPECT_EQ(survivors[epRank]->layer(1).getUnusableLocalSlots(), emptySpareSlots); + } +} + +TEST(MoeLoadBalancerRestoreTest, RestoreRejectsExtraReplicas) +{ + ScopedEnv const allowManagedFallback("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + using Placement = std::vector>; + constexpr int kFailedRank = 2; + // A single replica per expert and a spare slot per rank, so routing uses the single-replica kernel. + SimulatedEpRank rank( + /*epRank=*/0, /*epSize=*/3, /*expertCount=*/3, /*slotCountPerRank=*/2, /*layerCount=*/1, {0, -1, 1, -1, 2, -1}); + + // Expert 2 moves into rank 0's spare slot, and expert 1 gets a second replica in rank 1's. + Placement const redundant{{0, 2}, {1, 1}, {2, -1}}; + EXPECT_THROW(rank.balancer().restorePlacement({kFailedRank}, {redundant}), tensorrt_llm::common::TllmException); + EXPECT_EQ(rank.layer(0).getOldRankExpertIds(), (Placement{{0, -1}, {1, -1}, {2, -1}})); + + Placement const restored{{0, 2}, {1, -1}, {2, -1}}; + rank.loadSlots(0, restored); + rank.balancer().restorePlacement({kFailedRank}, {restored}); + EXPECT_EQ(rank.layer(0).getOldRankExpertIds(), (Placement{{0, 2}, {1, -1}, {-1, -1}})); + rank.expectSlotsHoldPlacement(0); +} + +TEST(MoeLoadBalancerRestoreTest, RestoreInstallsWithoutLoadingSlots) +{ + ScopedEnv const allowManagedFallback("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + using Placement = std::vector>; + SimulatedEpRank rank( + /*epRank=*/0, /*epSize=*/3, /*expertCount=*/3, /*slotCountPerRank=*/2, /*layerCount=*/1, {0, -1, 1, -1, 2, -1}); + + // Expert 2 moves into rank 0's spare slot, which the caller has not loaded: the slot keeps its bytes. + rank.balancer().restorePlacement({2}, {Placement{{0, 2}, {1, -1}, {2, -1}}}); + EXPECT_EQ(rank.layer(0).getOldRankExpertIds(), (Placement{{0, 2}, {1, -1}, {-1, -1}})); + EXPECT_EQ(rank.slotByte(0, 1), 0); +} + +// An online EPLB update copies the experts it moves into host accessible slots on the CPU, with and without fault +// tolerance. +TEST(MoeLoadBalancerCpuCopyTest, UpdateCopiesMovedExpertsIntoSlots) +{ + ScopedEnv const allowManagedFallback("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + constexpr size_t kWeightBytes = 4096; + constexpr int kExpertCount = 2; + constexpr int kSlotCount = 3; + // Expert 1 carries the load, so the update gives it a second slot. + std::vector const initialAssignments{0, 1, 0}; + std::vector const loads{1, 20}; + auto const expertByte = [](int expertId) { return static_cast(0x40 + expertId); }; + for (bool const faultTolerance : {false, true}) + { + MoeLoadBalancer balancer(/*epRank=*/0, /*epSize=*/1, /*layerUpdatesPerIter=*/1, faultTolerance); + auto& allocator = HostAccessibleDeviceAllocator::getInstance(); + if (!allocator.isSupported()) + { + GTEST_SKIP() << "Host accessible memory is not supported on this system."; + } + auto layer = balancer.AddLayer(kExpertCount, /*topK=*/1, kSlotCount); + std::vector hostWeights; + for (int expertId = 0; expertId < kExpertCount; ++expertId) + { + void* hostWeight = nullptr; + TLLM_CUDA_CHECK(cudaMallocHost(&hostWeight, kWeightBytes)); + std::fill_n(static_cast(hostWeight), kWeightBytes, expertByte(expertId)); + hostWeights.push_back(hostWeight); + layer->addSingleHostWeight(expertId, "weight", MoeWeight{hostWeight, 1, kWeightBytes, kWeightBytes}); + } + std::vector slots; + for (int slotId = 0; slotId < kSlotCount; ++slotId) + { + void* slot = allocator.allocate(kWeightBytes); + TLLM_CUDA_CHECK(cudaMemset(slot, expertByte(initialAssignments[slotId]), kWeightBytes)); + slots.push_back(slot); + layer->addSingleWeightSlot(slotId, "weight", MoeWeight{slot, 1, kWeightBytes, kWeightBytes}); + } + layer->setInitialWeightAssignments(initialAssignments); + balancer.finalizeModel(); + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + + balancer.startIter(0, /*enableStatistic=*/true, /*enableUpdateWeights=*/true); + runLayerOnGpu(*layer, 0, loads, /*handBack=*/true); + balancer.endIter(0); + // Waits for the weight update started. + balancer.shutdown(); + + auto const placement = layer->getOldRankExpertIds(); + EXPECT_NE(placement[0], initialAssignments) << "faultTolerance=" << faultTolerance; + for (int slotId = 0; slotId < kSlotCount; ++slotId) + { + std::vector bytes(kWeightBytes); + TLLM_CUDA_CHECK(cudaMemcpy(bytes.data(), slots[slotId], kWeightBytes, cudaMemcpyDeviceToHost)); + EXPECT_EQ(static_cast(std::count(bytes.begin(), bytes.end(), expertByte(placement[0][slotId]))), + kWeightBytes) + << "faultTolerance=" << faultTolerance << ", slot " << slotId << " should hold expert " + << placement[0][slotId]; + allocator.free(slots[slotId]); + } + for (auto* hostWeight : hostWeights) + { + EXPECT_EQ(cudaFreeHost(hostWeight), cudaSuccess); + } + } +} + +TEST(MoeLoadBalancerAbandonTest, AbandonIterationsWaitsForTheLaunchedWork) +{ + ScopedEnv const allowManagedFallback("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + // Shared with the thread that calls abandonIterations, so that a call that never returns fails + // the test instead of hanging it. + auto loadBalancer = std::make_shared( + /*epRank=*/0, /*epSize=*/2, /*layerUpdatesPerIter=*/1, /*faultTolerance=*/true); + loadBalancer->AddLayer(/*expertCount=*/2, /*topK=*/1, /*slotCountPerRank=*/2) + ->setInitialWeightAssignments({0, 1, 1, 0}); + loadBalancer->finalizeModel(); + // Device work that ends only when the test hands its layer over: a GPU stage wait on a signal of its own, in + // pinned host memory as a load balancer's signals are. + using tensorrt_llm::kernels::MoeLoadBalanceSingleLayerSignal; + MoeLoadBalanceSingleLayerSignal* held = nullptr; + TLLM_CUDA_CHECK(cudaMallocHost(&held, sizeof(*held))); + std::memset(static_cast(held), 0, sizeof(*held)); + held->stepAndOwner = MoeLoadBalanceSingleLayerSignal::kCPU; + int* enabled = nullptr; + TLLM_CUDA_CHECK(cudaMalloc(&enabled, sizeof(int))); + cudaStream_t stream = nullptr; + TLLM_CUDA_CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + tensorrt_llm::kernels::moeWaitSignalForGpuStageDevice(held, enabled, stream); + + std::promise abandonDone; + auto abandoned = abandonDone.get_future(); + std::thread( + [loadBalancer, abandonDone = std::move(abandonDone)]() mutable + { + std::exception_ptr error; + try + { + loadBalancer->abandonIterations(); + } + catch (...) + { + error = std::current_exception(); + } + // The test destroys the load balancer once the call returned. + loadBalancer.reset(); + if (error) + { + abandonDone.set_exception(error); + } + else + { + abandonDone.set_value(); + } + }) + .detach(); + EXPECT_EQ(abandoned.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout) + << "abandonIterations returned while launched work still ran"; + // Hand the layer over: the held work ends, and so does the call waiting for it. + tensorrt_llm::kernels::moeSetSignalForGpuStageHost(held, 0, /*enableStatistic=*/false); + if (abandoned.wait_for(std::chrono::seconds(10)) != std::future_status::ready) + { + FAIL() << "abandonIterations did not return once the launched work ended"; + } + abandoned.get(); + loadBalancer->shutdown(); + TLLM_CUDA_CHECK(cudaStreamDestroy(stream)); + TLLM_CUDA_CHECK(cudaFree(enabled)); + TLLM_CUDA_CHECK(cudaFreeHost(held)); +} + +namespace +{ + +//! Return the mapping granularity of unbacked slot memory on device 0. +size_t slotMemoryGranularity() +{ + CUmemAllocationProp prop{}; + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + prop.location.id = 0; + size_t granularity = 0; + TLLM_CU_CHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); + return granularity; +} + +//! Read the byte at offset of device memory. +uint8_t readByte(void const* ptr, size_t offset) +{ + uint8_t value = 0; + TLLM_CUDA_CHECK(cudaMemcpy(&value, static_cast(ptr) + offset, 1, cudaMemcpyDeviceToHost)); + return value; +} + +} // namespace + +TEST(UnbackedSlotMemoryTest, SavesOnlyChunksInsideUnbackedSlots) +{ + TLLM_CUDA_CHECK(cudaSetDevice(0)); + auto& memory = UnbackedSlotMemory::getInstance(); + size_t const granularity = slotMemoryGranularity(); + // Both slots share the one chunk, which the backed slot needs. + EXPECT_FALSE(memory.savesMemory(granularity / 2, 2, {1})); + // With slots that are not a multiple of the granularity, the last chunk holds only slot 1. + EXPECT_TRUE(memory.savesMemory(granularity + granularity / 2, 2, {1})); + // The tail of the last chunk belongs to no slot, so the chunk holds only slot 2. + EXPECT_TRUE(memory.savesMemory(granularity / 2, 3, {2})); + EXPECT_THROW(memory.savesMemory(granularity, 2, {2}), tensorrt_llm::common::TllmException); +} + +TEST(UnbackedSlotMemoryTest, SharedChunkStaysBackedWhileUsed) +{ + TLLM_CUDA_CHECK(cudaSetDevice(0)); + auto& memory = UnbackedSlotMemory::getInstance(); + size_t const granularity = slotMemoryGranularity(); + // Slots 2 and 3 share the second chunk. + void* ptr = memory.allocate(granularity / 2, 4, {2, 3}); + EXPECT_EQ(memory.bytesToBack(ptr, {2}), granularity); + EXPECT_EQ(memory.setBacked(ptr, {2}, true), granularity); + EXPECT_EQ(memory.bytesToBack(ptr, {3}), size_t{0}); + EXPECT_EQ(memory.setBacked(ptr, {3}, true), size_t{0}); + EXPECT_EQ(memory.setBacked(ptr, {2}, false), size_t{0}); + EXPECT_EQ(memory.setBacked(ptr, {3}, false), granularity); + EXPECT_EQ(memory.bytesToBack(ptr, {2, 3}), granularity); + EXPECT_THROW(memory.bytesToBack(ptr, {1}), tensorrt_llm::common::TllmException); + memory.free(ptr); +} + +TEST(UnbackedSlotMemoryTest, BackedSlotKeepsItsOwnBytes) +{ + TLLM_CUDA_CHECK(cudaSetDevice(0)); + auto& memory = UnbackedSlotMemory::getInstance(); + size_t const granularity = slotMemoryGranularity(); + void* ptr = memory.allocate(granularity, 3, {0, 2}); + auto* slots = static_cast(ptr); + // While unbacked, slots 0 and 2 map the same scratch. + TLLM_CUDA_CHECK(cudaMemset(slots, 0xAB, granularity)); + EXPECT_EQ(readByte(slots, 2 * granularity), 0xAB); + EXPECT_EQ(memory.setBacked(ptr, {0}, true), granularity); + TLLM_CUDA_CHECK(cudaMemset(slots, 0xCD, granularity)); + TLLM_CUDA_CHECK(cudaMemset(slots + 2 * granularity, 0, granularity)); + EXPECT_EQ(readByte(slots, 0), 0xCD); + memory.free(ptr); +} + +// Without fault tolerance a CPU-copied slot is kept at its host pointer, so an unbacked spare slot, which has no host +// mapping, counts as not added. With fault tolerance a CPU-copied slot is kept at its device address and an unbacked +// one counts as added. +TEST(MoeLoadBalancerCpuCopyTest, UnbackedSlotIsAddedOnlyWithFaultTolerance) +{ + ScopedEnv const forceGdrcopy("TRTLLM_EPLB_FORCE_GDRCOPY", "1"); + TLLM_CUDA_CHECK(cudaSetDevice(0)); + size_t const granularity = slotMemoryGranularity(); + for (bool const faultTolerance : {false, true}) + { + MoeLoadBalancer balancer(/*epRank=*/0, /*epSize=*/1, /*layerUpdatesPerIter=*/0, faultTolerance); + auto& allocator = HostAccessibleDeviceAllocator::getInstance(); + if (!allocator.isSupported() || !allocator.usesGdrCopy()) + { + GTEST_SKIP() << "Device memory is not mapped for the host with GDRCopy on this system."; + } + auto layer = balancer.AddLayer(/*expertCount=*/1, /*topK=*/1, /*slotCountPerRank=*/2); + void* hostWeight = nullptr; + TLLM_CUDA_CHECK(cudaMallocHost(&hostWeight, granularity)); + layer->addSingleHostWeight(0, "weight", MoeWeight{hostWeight, 1, granularity, granularity}); + auto& memory = UnbackedSlotMemory::getInstance(); + auto* slots = static_cast(memory.allocate(granularity, 2, {1}, /*hostAccessible=*/true)); + layer->addSingleWeightSlot(0, "weight", MoeWeight{slots, 1, granularity, granularity}); + layer->addSingleWeightSlot(1, "weight", MoeWeight{slots + granularity, 1, granularity, granularity}); + layer->setInitialWeightAssignments(faultTolerance ? std::vector{0, -1} : std::vector{0, 0}); + if (faultTolerance) + { + EXPECT_NO_THROW(balancer.finalizeModel()); + } + else + { + EXPECT_THROW(balancer.finalizeModel(), tensorrt_llm::common::TllmException); + } + memory.free(slots); + EXPECT_EQ(cudaFreeHost(hostWeight), cudaSuccess); + } +} + // Iteration control parameter structure struct IterConfig { diff --git a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py index 852b20873544..a00afd0864e0 100644 --- a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py @@ -995,6 +995,15 @@ def _(single_layer_load_balancer_ptr: int, token_selected_experts: torch.Tensor, offset_by_ep_rank: bool): return torch.empty_like(token_selected_experts) + @torch.library.register_fake("trtllm::moe_unbacked_slots_set_backed") + def _(tensors: List[torch.Tensor], slots: List[int], + slots_per_tensor: List[int], backed: bool) -> int: + return 0 + + @torch.library.register_fake("trtllm::moe_unbacked_slots_bytes_to_back") + def _(tensor: torch.Tensor, slots: List[int]) -> int: + return 0 + @torch.library.register_fake("trtllm::memset_expert_ids") def _(experts_ids: torch.Tensor, recv_rank_count_cumsum: torch.Tensor, max_token_count_per_rank: int, top_k: int, invalid_expert_id: int, diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py index c19500cbecd8..68290d608d81 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py @@ -276,7 +276,16 @@ def __init__(self, num_local_experts: int, local_expert_offset: int, tile_size: int, - seed: int = 515): + seed: int = 515, + tuner_num_local_experts: Optional[int] = None): + # num_experts, num_local_experts and local_expert_offset below describe + # the tuned layout, which has only the first tuner_num_local_experts + # slots of each rank. Row padding and expert-major capacities cover + # all runtime_num_local_experts slots that the kernels run over. + self.runtime_num_local_experts = num_local_experts + num_experts, num_local_experts, local_expert_offset = self.tuner_expert_layout( + num_experts, num_local_experts, local_expert_offset, + tuner_num_local_experts) self.num_experts = num_experts self.top_k = top_k self.num_local_experts = num_local_experts @@ -287,6 +296,18 @@ def __init__(self, # Intentionally use a large padding value to expose issues early. self.pad_val = int(2e9) + @staticmethod + def tuner_expert_layout( + num_experts: int, num_local_experts: int, local_expert_offset: int, + tuner_num_local_experts: Optional[int]) -> Tuple[int, int, int]: + """Return the expert layout that tunes only ``tuner_num_local_experts``.""" + if tuner_num_local_experts is None: + return num_experts, num_local_experts, local_expert_offset + ep_size = num_experts // num_local_experts + ep_rank = local_expert_offset // num_local_experts + return (ep_size * tuner_num_local_experts, tuner_num_local_experts, + ep_rank * tuner_num_local_experts) + def get_max_num_tiles(self, num_tokens: int) -> int: num_expanded_tokens = num_tokens * self.top_k if num_expanded_tokens <= self.num_local_experts: @@ -302,15 +323,15 @@ def get_expert_capacity_num_permuted_tokens(self, """Return padded rows for a fixed-capacity expert layout.""" if expert_capacity <= 0: raise ValueError("Expert capacity must be positive") - return (self.num_local_experts * + return (self.runtime_num_local_experts * ceil_div(expert_capacity, self.tile_size) * self.tile_size) def _expert_capacity_from_rows(self, num_rows: int) -> int: - if num_rows <= 0 or num_rows % self.num_local_experts != 0: + if num_rows <= 0 or num_rows % self.runtime_num_local_experts != 0: raise ValueError( "Expert-major rows must be positive and divisible by the " "number of local experts") - return num_rows // self.num_local_experts + return num_rows // self.runtime_num_local_experts def gen_expert_capacity_tuning_buckets(self, num_rows: int) -> Tuple[int, ...]: @@ -318,14 +339,15 @@ def gen_expert_capacity_tuning_buckets(self, capacity = max(self.tile_size, self._expert_capacity_from_rows(num_rows)) capacities = get_last_power_of_2_num_tokens_buckets(capacity) - return tuple(self.num_local_experts * value for value in capacities - if value >= self.tile_size) + return tuple(self.runtime_num_local_experts * value + for value in capacities if value >= self.tile_size) def map_to_expert_capacity_tuning_bucket(self, num_rows: int) -> int: """Map an expert-major layout to a stable power-of-two capacity bucket.""" capacity = max(self.tile_size, self._expert_capacity_from_rows(num_rows)) - return (self.num_local_experts * last_positive_power_of_2(capacity)) + return (self.runtime_num_local_experts * + last_positive_power_of_2(capacity)) def infer_shape_input0_rows(self, input_shapes: List[torch.Size]) -> int: return input_shapes[0][0] @@ -338,24 +360,33 @@ def infer_shape_padded_rows_from_input5( capacity = self._expert_capacity_from_rows(input_shapes[5][0]) return self.get_expert_capacity_num_permuted_tokens(capacity) - def infer_num_tokens(self, max_num_permuted_tokens: int) -> int: + def infer_num_tokens(self, + max_num_permuted_tokens: int, + num_local_experts: Optional[int] = None) -> int: """Infer the maximum possible number of tokens given the max_num_permuted_tokens. + + ``num_local_experts`` defaults to the tuned layout's. Pass + ``runtime_num_local_experts`` for the rows of every local slot. """ + if num_local_experts is None: + num_local_experts = self.num_local_experts max_num_tiles = max_num_permuted_tokens // self.tile_size - if max_num_tiles >= self.num_local_experts: + if max_num_tiles >= num_local_experts: return (max_num_permuted_tokens - (self.tile_size - 1) * - (self.num_local_experts - 1)) // self.top_k + (num_local_experts - 1)) // self.top_k return max_num_tiles // self.top_k def gen_tuning_buckets(self, max_num_tokens: int) -> List[int]: buckets = get_last_power_of_2_num_tokens_buckets( - self.infer_num_tokens(max_num_tokens)) + self.infer_num_tokens(max_num_tokens, + self.runtime_num_local_experts)) return sorted( list(set(self.get_max_num_permuted_tokens(x) for x in buckets))) def map_to_tuning_buckets(self, x: int) -> int: return self.get_max_num_permuted_tokens( - last_positive_power_of_2(self.infer_num_tokens(x))) + last_positive_power_of_2( + self.infer_num_tokens(x, self.runtime_num_local_experts))) def infer_shape_num_tokens(self, input_shapes: List[torch.Size]) -> int: return self.infer_num_tokens(input_shapes[self.IDX_SHAPE_INFER][0]) @@ -3090,13 +3121,15 @@ def __init__(self, local_expert_offset: int, tile_size: int, output_dtype: torch.dtype, - scaling_vector_size: int = 16): + scaling_vector_size: int = 16, + tuner_num_local_experts: Optional[int] = None): super().__init__() self.num_experts = num_experts self.top_k = top_k self.num_local_experts = num_local_experts self.local_expert_offset = local_expert_offset self.tile_size = tile_size + self.tuner_num_local_experts = tuner_num_local_experts assert output_dtype == torch.bfloat16 self.output_dtype = output_dtype @@ -3113,7 +3146,7 @@ def __init__(self, ) def unique_id(self): - return ( + runner_id = ( self.num_experts, self.top_k, self.num_local_experts, @@ -3122,6 +3155,10 @@ def unique_id(self): self.output_dtype, self.scaling_vector_size, ) + if self.tuner_num_local_experts is not None: + runner_id += ("tuner_num_local_experts", + self.tuner_num_local_experts) + return runner_id def get_valid_tactics( self, @@ -3171,10 +3208,13 @@ def get_valid_tactics( def get_tuning_config(self) -> TuningConfig: key = self.unique_id() if key not in self.__class__.tuning_config_cache: - helper = GroupedGemmInputsHelper(self.num_experts, self.top_k, - self.num_local_experts, - self.local_expert_offset, - self.tile_size) + helper = GroupedGemmInputsHelper( + self.num_experts, + self.top_k, + self.num_local_experts, + self.local_expert_offset, + self.tile_size, + tuner_num_local_experts=self.tuner_num_local_experts) self.__class__.tuning_config_cache[key] = TuningConfig( dynamic_tensor_specs=(DynamicTensorSpec( 0, 0, helper.gen_tuning_buckets, @@ -3334,12 +3374,14 @@ def cute_dsl_nvfp4_grouped_gemm_blackwell( tile_size: int, output_dtype: torch.dtype, scaling_vector_size: int = 16, + tuner_num_local_experts: Optional[int] = None, ) -> torch.Tensor: tuner = AutoTuner.get() runner = Sm100BlockScaledContiguousGroupedGemmRunner( num_experts, top_k, num_local_experts, local_expert_offset, - tile_size, output_dtype, scaling_vector_size) + tile_size, output_dtype, scaling_vector_size, + tuner_num_local_experts) inputs = [ input, weight, input_scale, weight_scale, alpha, tile_idx_to_group_idx, num_non_exiting_tiles @@ -3371,6 +3413,7 @@ def _( tile_size: int, output_dtype: torch.dtype, scaling_vector_size: int = 16, + tuner_num_local_experts: Optional[int] = None, ) -> torch.Tensor: m = input.size(0) n = weight.size(1) @@ -3392,7 +3435,8 @@ def __init__(self, scaling_vector_size: int = 16, use_expert_counts: bool = False, weights_evict_first: bool = False, - weights_evict_first_max_tokens: int = 0): + weights_evict_first_max_tokens: int = 0, + tuner_num_local_experts: Optional[int] = None): """``weights_evict_first``: load the expert weights (B / SFB) evict-first. Such a runner serves calls of up to ``weights_evict_first_max_tokens`` tokens (the caller's bound; 0: unknown), and its tuning covers row buckets up to that bound whatever the tuning call's rows.""" @@ -3402,6 +3446,7 @@ def __init__(self, self.num_local_experts = num_local_experts self.local_expert_offset = local_expert_offset self.tile_size = tile_size + self.tuner_num_local_experts = tuner_num_local_experts assert output_dtype == torch.bfloat16 self.output_dtype = output_dtype @@ -3444,6 +3489,9 @@ def unique_id(self): ) if self.weights_evict_first: runner_id += ("weights_evict_first", ) + if self.tuner_num_local_experts is not None: + runner_id += ("tuner_num_local_experts", + self.tuner_num_local_experts) return runner_id def _get_default_tactic( @@ -3512,10 +3560,13 @@ def get_tuning_config(self) -> TuningConfig: that bound (``tune_max_num_tokens``), so its tactics cover every call it serves.""" key = (self.unique_id(), self.weights_evict_first_max_tokens) if key not in self.__class__.tuning_config_cache: - helper = GroupedGemmInputsHelper(self.num_experts, self.top_k, - self.num_local_experts, - self.local_expert_offset, - self.tile_size) + helper = GroupedGemmInputsHelper( + self.num_experts, + self.top_k, + self.num_local_experts, + self.local_expert_offset, + self.tile_size, + tuner_num_local_experts=self.tuner_num_local_experts) if self.use_expert_counts: self.__class__.tuning_config_cache[key] = TuningConfig( dynamic_tensor_specs=(DynamicTensorSpec( @@ -3778,6 +3829,7 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( expert_capacity: int = 0, weights_evict_first: bool = False, weights_evict_first_max_tokens: int = 0, + tuner_num_local_experts: Optional[int] = None, ) -> None: """``weights_evict_first``: load the expert weights (B / SFB) with an L2 evict_first policy (same math). ``weights_evict_first_max_tokens``: the most tokens of a call that takes ``weights_evict_first`` (the @@ -3806,7 +3858,8 @@ def cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell( scaling_vector_size, use_expert_counts=expert_counts is not None, weights_evict_first=weights_evict_first, - weights_evict_first_max_tokens=weights_evict_first_max_tokens) + weights_evict_first_max_tokens=weights_evict_first_max_tokens, + tuner_num_local_experts=tuner_num_local_experts) inputs = [ input, weight, input_scale, weight_scale, alpha, output, @@ -3848,6 +3901,7 @@ def _( expert_capacity: int = 0, weights_evict_first: bool = False, weights_evict_first_max_tokens: int = 0, + tuner_num_local_experts: Optional[int] = None, ) -> None: num_tokens = token_final_scales.size(0) n = weight.size(1) @@ -4287,7 +4341,8 @@ def __init__(self, swiglu_limit_scalar: float = float("inf"), use_expert_counts: bool = False, situ_beta: Optional[float] = None, - situ_linear_beta: Optional[float] = None): + situ_linear_beta: Optional[float] = None, + tuner_num_local_experts: Optional[int] = None): """Initialize the runner. Args: @@ -4298,6 +4353,10 @@ def __init__(self, situ_beta: Gate-side SiTU soft-cap. Required for -- and only valid with -- ``ActivationType.SiTu``. situ_linear_beta: Linear-side SiTU soft-cap, same rule. + tuner_num_local_experts: If set, profile only the first + ``tuner_num_local_experts`` local slots of each rank. The + kernel still runs over every local slot (see + ``GroupedGemmInputsHelper``). """ super().__init__() self.activation_type = validate_activation_type(activation_type) @@ -4308,6 +4367,7 @@ def __init__(self, self.top_k = top_k self.num_local_experts = num_local_experts self.local_expert_offset = local_expert_offset + self.tuner_num_local_experts = tuner_num_local_experts if tile_size not in [128, 256]: raise ValueError( f"Tile size {tile_size} is not supported, it only supports 128 and 256." @@ -4337,9 +4397,11 @@ def unique_id(self): so two runners that differ in any of them are different kernels and must not share a tuning result. That is why the activation soft-caps appear: ``swiglu_limit_scalar`` and the two SiTU betas - are baked in as ``const_expr``, not passed at launch. + are baked in as ``const_expr``, not passed at launch. A tuner + layout of fewer local experts changes what is profiled, so it + separates the tuning results as well. """ - return ( + runner_id = ( self.num_experts, self.top_k, self.num_local_experts, @@ -4352,6 +4414,10 @@ def unique_id(self): self.situ_beta, self.situ_linear_beta, ) + if self.tuner_num_local_experts is not None: + runner_id += ("tuner_num_local_experts", + self.tuner_num_local_experts) + return runner_id def get_valid_tactics( self, @@ -4439,6 +4505,7 @@ def get_tuning_config(self) -> TuningConfig: self.num_local_experts, self.local_expert_offset, self.tile_size, + tuner_num_local_experts=self.tuner_num_local_experts, ) # Tuning uses layout: # a, b, a_sf, b_sf, alpha, tile_idx, tile_mn_limit, permuted_idx, ... @@ -4740,6 +4807,7 @@ def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( expert_capacity: int = 0, situ_beta: Optional[float] = None, situ_linear_beta: Optional[float] = None, + tuner_num_local_experts: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """CuteDSL-based NVFP4 gather grouped GEMM with activation fusion. @@ -4777,7 +4845,8 @@ def cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell( swiglu_limit_scalar=swiglu_limit_scalar, use_expert_counts=expert_counts is not None, situ_beta=situ_beta, - situ_linear_beta=situ_linear_beta) + situ_linear_beta=situ_linear_beta, + tuner_num_local_experts=tuner_num_local_experts) inputs = [ input, weight, input_scale, weight_scale, alpha, tile_idx_to_group_idx, tile_idx_to_mn_limit, @@ -4910,6 +4979,7 @@ def _fake_single_b( expert_capacity: int = 0, situ_beta: Optional[float] = None, situ_linear_beta: Optional[float] = None, + tuner_num_local_experts: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """Meta-device shapes for the FC1 output and its block scales. diff --git a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py index 0f88db7f9680..f21c388e12bc 100644 --- a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py @@ -126,6 +126,7 @@ def __init__( activation_type: ActivationType, unpadded_hidden_size: Optional[int] = None, use_mxfp8_weight_scaling: bool = False, + tuner_num_local_experts: Optional[int] = None, ): self.x_dtype = x_dtype self.weight_dtype = weight_dtype @@ -148,6 +149,7 @@ def __init__( self.use_fused_finalize = use_fused_finalize self.activation_type = activation_type self.unpadded_hidden_size = unpadded_hidden_size if unpadded_hidden_size is not None else 0 + self.tuner_num_local_experts = tuner_num_local_experts instance_key = (x_dtype, weight_dtype, output_dtype, use_deepseek_fp8_block_scale, use_w4_group_scaling, @@ -178,7 +180,7 @@ def _resolve_fallback_tactic(self, tactic: int, gemm_idx: int) -> int: return tactic def unique_id(self): - return ( + identity = ( self.x_dtype, self.weight_dtype, self.output_dtype, @@ -197,6 +199,9 @@ def unique_id(self): self.unpadded_hidden_size, self.use_mxfp8_weight_scaling, ) + if self.tuner_num_local_experts is not None: + identity += (self.tuner_num_local_experts, ) + return identity def forward( self, @@ -206,6 +211,11 @@ def forward( do_preparation: bool = False, ): x, fc1_expert_weights, fc1_expert_biases, fc2_expert_weights, fc2_expert_biases = inputs + if self.tuner_num_local_experts is not None: + # The profiler routes tokens to every expert of the weights it gets. + fc1_expert_weights, fc1_expert_biases, fc2_expert_weights, fc2_expert_biases = ( + None if w is None else w[:self.tuner_num_local_experts] + for w in inputs[1:]) tactic = self._resolve_fallback_tactic(tactic, gemm_idx) self.fused_moe_runner.run_gemm_profile( x, @@ -291,6 +301,7 @@ def fused_moe( gated_slot_lora_weight_ptrs: Optional[torch.Tensor] = None, token_to_slot: Optional[torch.Tensor] = None, swiglu_clamp_after_silu: bool = False, + tuner_num_local_experts: Optional[int] = None, ) -> List[torch.Tensor]: tuner = AutoTuner.get() # Only the non-alltoall case is considered for profiling in the warmup phase. @@ -326,6 +337,7 @@ def fused_moe( activation_type=activation_type, unpadded_hidden_size=unpadded_hidden_size, use_mxfp8_weight_scaling=use_mxfp8_weight_scaling, + tuner_num_local_experts=tuner_num_local_experts, ) MoERunner.tuning_config.tune_max_num_tokens = tune_max_num_tokens @@ -468,7 +480,8 @@ def _(input: torch.Tensor, gated_slot_lora_ranks: Optional[torch.Tensor] = None, gated_slot_lora_weight_ptrs: Optional[torch.Tensor] = None, token_to_slot: Optional[torch.Tensor] = None, - swiglu_clamp_after_silu: bool = False): + swiglu_clamp_after_silu: bool = False, + tuner_num_local_experts: Optional[int] = None): seq_len = input.shape[0] if use_int8_woq_per_channel: # Note: The weight shape for INT8 weight only quantization is different, i.e., diff --git a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md index 0055a0447378..6d9ab45ffb2c 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -255,6 +255,8 @@ wakeup. Under `enable_wide_ep_fault_tolerance`, `CommunicationFactory` builds `NVLinkOneSided` with its rank-mask kernels, and the model loader binds every MoE layer's communication to one `MoeExecutionState` (`communication/execution_state.py`, bound by `pyexecutor/rank_failure_model_load.py`) before the weights load. The kernels exchange only with the EP ranks set in the state's device mask, which they read when they run, so captured CUDA graphs follow a mask that `install_mask` rewrites while no collective runs. They stop waiting on peers once the state's pinned host abort state is requested (`request_abort`), and a wait on a peer longer than about 10 ms publishes the round flag it waits for there (`waited_flags`). A collective that ends without some active peer marks its round incomplete in the workspace, so later collectives exchange with this rank only until `rearm` resets the workspaces. These layers set up no sentinel transport, whose kernels take no rank mask. +A MoE backend runs with the unbacked spare EPLB slots of fault tolerance only if it declares `MoEStaticCapability.supports_spare_slots`: its weight method creates the spare slots and its autotuner profiles only the active slots (`tuner_layout_kwargs`). The Cutlass unquantized and NVFP4 leaves and the CuteDSL NVFP4 grouped-GEMM leaf declare it. While the load balancer adds spare slots (`MoEDeployment.spare_slots_enabled`), selection turns down every other candidate that `can_implement` accepts with `MoERejectReason.SPARE_SLOTS_UNSUPPORTED`, as it does for EPLB, and so does the CuteDSL leaf's `can_implement` on SM107, whose kernels tune every local slot. A backend literal then falls back like any degradation, to a Cutlass leaf with the capability, but the warning, which names Wide-EP fault tolerance and the reason, is logged once per process rather than once per layer; a pinned tactic, or a layer whose format no leaf with the capability serves, fails with that reason. + ### Core Design Principles 1. **Composition over inheritance** — Backend, Communication, EPLB, and Scheduler are independent, composable components diff --git a/tensorrt_llm/_torch/moe/fused_moe/create_moe.py b/tensorrt_llm/_torch/moe/fused_moe/create_moe.py index 074e8a8f2a31..0b8ff6510cae 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/create_moe.py +++ b/tensorrt_llm/_torch/moe/fused_moe/create_moe.py @@ -114,13 +114,19 @@ def create_moe_backend( # Backstop for the direct-``moe_cls`` callers that bypass resolution: a # backend that cannot run under a load balancer already declines in # ``can_implement``. - eplb_enabled = get_moe_load_balancer() is not None + balancer = get_moe_load_balancer() + eplb_enabled = balancer is not None load_balancer_config = getattr(model_config, "moe_load_balancer", None) eplb_mode = (getattr(load_balancer_config, "mode", "standard") if eplb_enabled else "disabled") if eplb_enabled and not moe_cls.capabilities.supports_eplb: raise ValueError( f"{moe_cls.__name__} does not support the MoE load balancer.") + if (eplb_enabled and balancer.adds_spare_slots + and not moe_cls.capabilities.supports_spare_slots): + raise ValueError( + f"{moe_cls.__name__} does not support the unbacked spare EPLB slots " + "of Wide-EP fault tolerance.") if (eplb_mode == "per_layer_dynamic" and not moe_cls.capabilities.supports_per_layer_dynamic_eplb): raise ValueError( diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/eligibility.py b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/eligibility.py index fa33218935c2..16ac64f816a2 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/eligibility.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/eligibility.py @@ -21,6 +21,7 @@ MoEProblem, MoERejectReason, check_quant_matches_identity, + spare_slots_rejection, ) from ...impl_environment import MoEDep from ...interface import _reject @@ -43,6 +44,7 @@ def check_grouped_gemm_leaf( check_sm(cls, d), check_dtype(cls, p), check_grouped_gemm_shared(cls, p, d), + spare_slots_rejection(cls, d), *leaf_gates, ): if verdict is not None: diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/nvfp4.py b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/nvfp4.py index b045fe243317..c31db25e62ab 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/nvfp4.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/grouped_gemm/nvfp4.py @@ -15,6 +15,7 @@ """``TrtllmCutedslGroupedGemmNvfp4Impl``: ``trtllm.cutedsl.grouped_gemm.nvfp4``.""" import os +from functools import cached_property, partial import torch @@ -32,6 +33,7 @@ MoEFusedRouteQuant, MoEInputRequirement, MoEProblem, + MoERejectReason, MoERunContext, MoESortTables, MoEStaticCapability, @@ -86,6 +88,9 @@ class TrtllmCutedslGroupedGemmNvfp4Impl(CuteDslFusedMoEBase): supports_eplb=True, supports_apply_router_weight_on_input=True, supports_deep_ep_direct_metadata=True, + # Not on SM107 (see can_implement) or with locality domains, which + # decline EPLB. + supports_spare_slots=True, ), input_requirement=MoEInputRequirement(routing_scales_dtype=torch.float32), doc="CuTe DSL NVFP4 grouped GEMM: gather + FC1 + fused activation, " @@ -94,6 +99,42 @@ class TrtllmCutedslGroupedGemmNvfp4Impl(CuteDslFusedMoEBase): sm_support = SmSupport(allowed=frozenset({100, 103, 107})) _locality_domain_full_scale_params = ("w3_w1_weight_scale", "w2_weight_scale") + def create_weights(self) -> None: + if self._weights_created: + return + super().create_weights() + if self.uses_spare_slots: + # The GEMMs and their tuner profile only the active slots. + layout = self.tuner_layout_kwargs + self._fused_moe_runner = partial(CuteDslFusedMoENvfp4Runner, **layout) + self._gather_act_fusion_op = partial( + torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell, **layout + ) + self._finalize_inplace_op = partial( + torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell, **layout + ) + self._grouped_gemm_op = partial( + torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_blackwell, **layout + ) + + # The runner and the Blackwell ops of the forward. A module with spare slots binds the tuner + # layout of its active slots to them in create_weights; the Rubin kernels take no spare slots. + @cached_property + def _fused_moe_runner(self): + return CuteDslFusedMoENvfp4Runner + + @cached_property + def _gather_act_fusion_op(self): + return torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell + + @cached_property + def _finalize_inplace_op(self): + return torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell + + @cached_property + def _grouped_gemm_op(self): + return torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_blackwell + # Kinds mirror the act-fusion kernels' own SUPPORTED_ACTIVATION_TYPES. # There are two of them and ``run_moe_nvfp4`` picks between them by SM: # moe/kernels/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py @@ -110,10 +151,19 @@ class TrtllmCutedslGroupedGemmNvfp4Impl(CuteDslFusedMoEBase): @classmethod def can_implement(cls, p: MoEProblem, d: MoEDeployment) -> MoEEligibility: + spare_slots = None + if d.spare_slots_enabled and d.env.sm == 107: + spare_slots = MoEEligibility.no( + MoERejectReason.SPARE_SLOTS_UNSUPPORTED, + f"Wide-EP fault tolerance: {cls.__name__} on SM107 does not support spare EPLB " + "slots, as it tunes every local slot", + ) # process_weights_after_loading() unswizzles the FC1 block scales, # which asserts 128-row tiles; without this gate an unaligned shard # dies mid weight load with a bare swizzle error. - return check_grouped_gemm_leaf(cls, p, d, nvfp4_fc1_row_alignment_rejection(p, d)) + return check_grouped_gemm_leaf( + cls, p, d, nvfp4_fc1_row_alignment_rejection(p, d), spare_slots + ) def _init_format_state(self) -> None: # Opt-out for adapter-free DeepEP metadata; see @@ -375,7 +425,7 @@ def run_moe_nvfp4( ] tuner = AutoTuner.get() - runner = CuteDslFusedMoENvfp4Runner( + runner = self._fused_moe_runner( forward_impl=forward_impl, num_experts=self.num_slots, top_k=effective_top_k, @@ -539,7 +589,7 @@ def run_moe_nvfp4_impl( gather_act_op = ( torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_rubin if use_rubin - else torch.ops.trtllm.cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell + else self._gather_act_fusion_op ) gather_act_kwargs = dict( @@ -644,7 +694,7 @@ def run_moe_nvfp4_impl( finalize_inplace_op = ( torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_rubin if use_rubin - else torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell + else self._finalize_inplace_op ) finalize_inplace_kwargs = dict( @@ -700,7 +750,7 @@ def run_moe_nvfp4_impl( "Basic grouped GEMM without finalize fusion is not yet " "supported on Rubin." ) - x = torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_blackwell( + x = self._grouped_gemm_op( input=x.view(torch.float4_e2m1fn_x2), weight=weight_view.w2_weight.view(torch.float4_e2m1fn_x2), input_scale=x_sf.view(torch.uint8), diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/runners.py b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/runners.py index aaf4a0d87cd4..f50cefdf91c3 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutedsl/runners.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutedsl/runners.py @@ -97,7 +97,12 @@ def __init__( num_local_experts: int, local_expert_offset: int, spread_num_tokens: bool = False, + tuner_num_local_experts: int | None = None, ) -> None: + # Generated routing passes through this rank's runtime moe_sort, so keep the runtime offset. + num_experts, num_local_experts, _ = self.tuner_expert_layout( + num_experts, num_local_experts, local_expert_offset, tuner_num_local_experts + ) self.num_experts = num_experts self.top_k = top_k self.num_local_experts = num_local_experts @@ -172,6 +177,7 @@ def __init__( workload_identity: tuple | None = None, prime_inner_tactics: bool = False, spread_tuning_tokens: bool = False, + tuner_num_local_experts: int | None = None, ) -> None: super().__init__() self.forward_impl = forward_impl @@ -179,6 +185,7 @@ def __init__( self.top_k = top_k self.num_local_experts = num_local_experts self.local_expert_offset = local_expert_offset + self.tuner_num_local_experts = tuner_num_local_experts self.enable_finalize_fusion = enable_finalize_fusion self.enable_alltoall = enable_alltoall self.use_direct_expert_metadata = use_direct_expert_metadata @@ -213,6 +220,8 @@ def unique_id(self) -> tuple: identity += (self.workload_identity,) if self.use_direct_expert_metadata: identity += ("direct_expert_metadata",) + if self.tuner_num_local_experts is not None: + identity += ("tuner_num_local_experts", self.tuner_num_local_experts) return identity def get_valid_tactics( @@ -276,6 +285,7 @@ def get_tuning_config(self) -> TuningConfig: self.num_local_experts, self.local_expert_offset, spread_num_tokens=self.spread_tuning_tokens, + tuner_num_local_experts=self.tuner_num_local_experts, ) tuning_config = TuningConfig( dynamic_tensor_specs=( diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutlass/base.py b/tensorrt_llm/_torch/moe/fused_moe/cutlass/base.py index 267fcad781d8..d904da147364 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutlass/base.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutlass/base.py @@ -22,6 +22,7 @@ from collections.abc import Mapping from dataclasses import dataclass, fields +from functools import cached_property, partial import torch @@ -251,6 +252,14 @@ def create_weights(self) -> None: return self._check_quant_config_is_my_format() super().create_weights() + if self.uses_spare_slots: + self._fused_moe_op = partial(torch.ops.trtllm.fused_moe, **self.tuner_layout_kwargs) + + @cached_property + def _fused_moe_op(self): + """``torch.ops.trtllm.fused_moe``. A module with spare slots binds the tuner layout of its + active slots to it in :meth:`create_weights`.""" + return torch.ops.trtllm.fused_moe def _check_configs(self) -> None: assert self._weights_created @@ -349,7 +358,7 @@ def _run_fused_moe_op( w3_w1, w2 = dequantized_weights quant_scales, input_sf, swizzled_input_sf = [], None, False - result = torch.ops.trtllm.fused_moe( + result = self._fused_moe_op( ctx.x, ctx.token_selected_experts, ctx.token_final_scales, diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutlass/eligibility.py b/tensorrt_llm/_torch/moe/fused_moe/cutlass/eligibility.py index 80cf775dedac..46778cf566bd 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutlass/eligibility.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutlass/eligibility.py @@ -30,6 +30,7 @@ MoEProblem, MoERejectReason, check_quant_matches_identity, + spare_slots_rejection, ) from ..interface import _reject @@ -181,6 +182,7 @@ def check_cutlass_leaf( check_dtype(cls, p), check_gptoss_activation(cls, p), check_moe_lora(cls, d), + spare_slots_rejection(cls, d), *leaf_gates, check_oss_cutlass_build(cls, p, d), ): diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutlass/identity.py b/tensorrt_llm/_torch/moe/fused_moe/cutlass/identity.py index eb6b97c526f6..bcfceb784548 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutlass/identity.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutlass/identity.py @@ -59,6 +59,7 @@ def cutlass_descriptor( kernel_name: str = KERNEL_GROUPED_GEMM, supports_moe_lora: bool = False, supports_expert_bias: bool = True, + supports_spare_slots: bool = False, ) -> MoEImplDescriptor: """Build one leaf's descriptor. @@ -76,6 +77,10 @@ def cutlass_descriptor( in ``CutlassMoELoraMixin`` may set it. supports_expert_bias: Whether the leaf's kernel applies an expert FC bias. + supports_spare_slots: Whether the leaf runs with the unbacked spare + EPLB slots of Wide-EP fault tolerance: its weight method creates + them, and ``torch.ops.trtllm.fused_moe`` tunes only the active + slots. """ return MoEImplDescriptor( identity=MoEImplId(provider, technique, kernel_name, quant), @@ -84,6 +89,7 @@ def cutlass_descriptor( CUTLASS_CAPABILITIES, supports_moe_lora=supports_moe_lora, supports_expert_bias=supports_expert_bias, + supports_spare_slots=supports_spare_slots, ), input_requirement=CUTLASS_INPUT_REQUIREMENT, doc=doc, diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutlass/nvfp4.py b/tensorrt_llm/_torch/moe/fused_moe/cutlass/nvfp4.py index 594f8ef0ad25..2f50c566da85 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutlass/nvfp4.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutlass/nvfp4.py @@ -46,6 +46,7 @@ class TrtllmCutlassNvfp4Impl(CutlassFusedMoEBase): descriptor = cutlass_descriptor( "nvfp4", "CUTLASS grouped GEMM over NVFP4 weights and activations, Blackwell.", + supports_spare_slots=True, ) sm_support = SmSupport(allowed=frozenset({100, 103, 107, 120, 121})) supported_dtypes = HP_DTYPES_WITH_FP8 diff --git a/tensorrt_llm/_torch/moe/fused_moe/cutlass/unquantized.py b/tensorrt_llm/_torch/moe/fused_moe/cutlass/unquantized.py index 4af206ab45f8..9b689a70eea1 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/cutlass/unquantized.py +++ b/tensorrt_llm/_torch/moe/fused_moe/cutlass/unquantized.py @@ -35,6 +35,7 @@ class TrtllmCutlassUnquantizedImpl(CutlassMoELoraMixin, CutlassFusedMoEBase): "none", "CUTLASS grouped GEMM over unquantized fp16/bf16 weights, SM80+.", supports_moe_lora=True, + supports_spare_slots=True, ) sm_support = SmSupport(minimum=80) supported_dtypes = HP_DTYPES diff --git a/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py new file mode 100644 index 000000000000..df1f0abbe5e6 --- /dev/null +++ b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py @@ -0,0 +1,549 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Restore a failed rank's experts into EPLB spare slots under Wide-EP fault tolerance. + +Every node keeps host weights for all experts. After a failure, every active rank plans the same +placement and loads the experts it receives from its node's host weights. +""" + +import collections +import hashlib +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Optional + +import numpy as np +import torch + +from tensorrt_llm._wide_ep_fault_tolerance import TURN_FAULT_TOLERANCE_OFF +from tensorrt_llm.bindings.internal import runtime +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.math_utils import ceil_div + +from .moe_load_balancer import HostMoeTensorSharer, MoeLoadBalancer, SingleLayerMoeLoadBalancer + + +def plan_layer_placement( + current: np.ndarray, + base: np.ndarray, + active: np.ndarray, + active_slot_count: int, + expert_count: int, + layer_idx: int, +) -> np.ndarray: + """Return a deterministic ``[ranks, slots]`` placement of every expert on the ``active`` ranks. + + Active ranks keep their active slots from ``base`` and their spare slots from ``current``. An + active slot that repeats an expert of an earlier active slot holds an extra replica, which an + interrupted online EPLB update can leave behind. A lost expert takes an emptied active slot + first. Next it takes an extra replica's slot if routing uses one replica per expert, else an + empty spare slot, and an extra replica's slot only when no spare slot is left. + + Args: + current: ``[ranks, slots]`` experts, -1 for an empty slot. + base: ``current``, or the initial placement when a rank rejoins. + active: ``[ranks]`` flags telling whether a rank is active. + active_slot_count: The active slots per rank, the slots before the spare slots. + expert_count: The number of experts of the layer. + layer_idx: The layer index, which rotates the rank order so restored experts spread out. + + Raises: + ValueError: The active ranks' spare slots and extra replicas cannot hold every expert. + """ + placement = np.full_like(current, -1) + placement[active, :active_slot_count] = base[active, :active_slot_count] + active_ranks = np.flatnonzero(active) + # The empty active slots and the active slots of extra replicas, in rank and slot order. + empty_slots, extra_slots = collections.deque(), collections.deque() + kept = np.zeros(expert_count, dtype=bool) + for rank in active_ranks: + for slot in range(active_slot_count): + expert_id = placement[rank, slot] + if expert_id < 0: + empty_slots.append((rank, slot)) + elif kept[expert_id]: + extra_slots.append((rank, slot)) + else: + kept[expert_id] = True + single_replica = active_slot_count * current.shape[0] == expert_count + if single_replica: + for rank, slot in extra_slots: + placement[rank, slot] = -1 + replicas = np.bincount(placement[placement >= 0], minlength=expert_count) + spare_slots = range(active_slot_count, current.shape[1]) + for rank in active_ranks: + for slot in spare_slots: + expert_id = current[rank, slot] + if expert_id >= 0 and replicas[expert_id] == 0: + placement[rank, slot] = expert_id + replicas[expert_id] = 1 + rank_order = np.roll(active_ranks, -layer_idx) + for expert_id in np.flatnonzero(replicas == 0): + replicas[expert_id] = 1 + if empty_slots: + placement[empty_slots.popleft()] = expert_id + continue + # Prefer the fewest filled spare slots, then the rank order, then a slot that held an expert + # in ``current`` (its memory may still be backed), then the lowest slot. + candidates = [ + ( + np.count_nonzero(placement[rank, active_slot_count:] >= 0), + order, + current[rank, slot] < 0, + slot, + ) + for order, rank in enumerate(rank_order) + for slot in spare_slots + if placement[rank, slot] < 0 + ] + if extra_slots and (single_replica or not candidates): + placement[extra_slots.popleft()] = expert_id + elif candidates: + _, order, _, slot = min(candidates) + placement[rank_order[order], slot] = expert_id + else: + raise ValueError( + f"The spare slots of active ranks {active_ranks.tolist()} cannot hold expert " + f"{expert_id}" + ) + return placement + + +@dataclass(frozen=True, eq=False) +class ExpertRestorePlan: + """Per-layer placements for one set of active ranks. + + Every field except ``device_bytes`` is the same on every active rank. + """ + + active_ranks: tuple[bool, ...] + placements: tuple[np.ndarray, ...] + placement_before: tuple[np.ndarray, ...] + digest: str + device_bytes: int + + +def _bytes(tensor: torch.Tensor) -> torch.Tensor: + """Return a flat uint8 view of the contiguous ``tensor``.""" + return tensor.view(-1).view(torch.uint8) + + +def _phase_done(times: Optional[dict[str, float]], phase: str, start_s: float) -> float: + """Record the milliseconds since ``start_s`` as ``phase`` in ``times`` and return the time.""" + now_s = time.perf_counter() + if times is not None: + times[phase] = (now_s - start_s) * 1e3 + return now_s + + +def create_spare_slot_storage( + layer: SingleLayerMoeLoadBalancer, name: str, slots: torch.Tensor +) -> Optional[torch.Tensor]: + """Return storage for ``name`` whose spare slots map no memory, or None if that saves none.""" + slot_bytes = slots.numel() // slots.shape[0] * slots.element_size() + spare_slots = list(layer.spare_slots) + if not torch.ops.trtllm.moe_unbacked_slots_save_memory(slot_bytes, slots.shape[0], spare_slots): + return None + storage = torch.ops.trtllm.moe_unbacked_slot_tensor( + list(slots.shape), slots.dtype, spare_slots, layer.updates_enabled + ) + if slots.device.type != "meta": + active = layer.active_slot_count_per_rank + storage[:active].copy_(slots[:active]) + layer.unbacked_slot_tensors[name] = storage + return storage + + +def slot_parameters(module: torch.nn.Module) -> list[tuple[str, torch.nn.Parameter]]: + """Return the named weights of ``module`` that have one row per local slot. + + Restoring an expert into a spare slot needs host weights for each of them. + ``register_weight_slots_after_to_cuda`` checks this. + """ + return [ + (name, param) + for name, param in module.named_parameters(recurse=False) + if param.dim() > 0 and param.shape[0] == module.expert_size_per_partition + ] + + +def create_spare_slot_weights(module: torch.nn.Module) -> None: + """Move the slot weights of ``module`` to unbacked spare slots.""" + layer = module.layer_load_balancer + for name, param in slot_parameters(module): + storage = create_spare_slot_storage(layer, name, param) + if storage is not None: + module.register_parameter( + name, torch.nn.Parameter(storage, requires_grad=param.requires_grad) + ) + + +def use_spare_slot_storage(layer: SingleLayerMoeLoadBalancer, wt: torch.Tensor, name: str) -> None: + """Take the loaded ``wt`` as the slots of weight ``name``.""" + unbacked = layer.unbacked_slot_tensors.get(name) + if unbacked is None: + if layer.updates_enabled: + layer.make_tensor_host_accessible(wt) + elif wt.untyped_storage().data_ptr() != unbacked.untyped_storage().data_ptr(): + raise RuntimeError( + f"Weight {name} of MoE layer {layer.get_layer_idx()} lost the storage whose spare " + "slots map no device memory: a transform after loading replaced it" + ) + layer.slot_tensors[name] = wt + + +def filled_spare_slots(layer: SingleLayerMoeLoadBalancer, local: np.ndarray) -> list[int]: + """Return the spare slots of ``layer`` that ``local`` fills.""" + return [slot for slot in layer.spare_slots if local[slot] >= 0] + + +def spare_slot_bytes_to_back(layer: SingleLayerMoeLoadBalancer, slots: Sequence[int]) -> int: + """Return the device bytes backing ``slots`` of ``layer`` would map now.""" + if not slots: + return 0 + return sum( + torch.ops.trtllm.moe_unbacked_slots_bytes_to_back(tensor, list(slots)) + for tensor in layer.unbacked_slot_tensors.values() + ) + + +class DeploymentHostWeights(HostMoeTensorSharer): + """Host tensor sharer that keeps the host weights on this node, which restores load slots from.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # The host weights on this node by expert and weight name, which restores load slots from. + self.node_host_weights: dict[tuple[int, str], torch.Tensor] = {} + + def finalize_host_tensor_sharing(self, add_host_weight_fn: Optional[Callable] = None): + def keep(expert_id: int, name: str, weight: torch.Tensor) -> None: + self.node_host_weights[(expert_id, name)] = weight + if add_host_weight_fn is not None: + add_host_weight_fn(expert_id, name, weight) + + super().finalize_host_tensor_sharing(keep) + + def pre_shutdown_cleanup(self): + # Shared memory closes only once no view of it is left. + self.node_host_weights.clear() + super().pre_shutdown_cleanup() + + +class EplbExpertRestore: + """Size the spare slots of a ``MoeLoadBalancer`` and restore failed ranks' experts into them.""" + + def __init__(self, balancer: "MoeLoadBalancer"): + self.balancer = balancer + nodes = self._rank_nodes() + self.largest_failure = max(map(len, self._recoverable_failures(nodes)), default=0) + if ( + self.largest_failure + and balancer.layer_updates_per_iter > 0 + and not runtime.host_accessible_device_memory_uses_gdrcopy() + ): + raise ValueError( + "Wide-EP fault tolerance with online EPLB requires GDRCopy in CDMM mode, " + "because online EPLB's CPU copies reach the spare slots only through GDRCopy. Set " + f"layer_updates_per_iter=0 or {TURN_FAULT_TOLERANCE_OFF}." + ) + self.active_ranks: tuple[bool, ...] = (True,) * balancer.ep_size + self.backed_spare_slot_bytes = 0 + # By layer, the slots of this rank that a restore wrote and no installed placement names yet. + self._invalid_slots: dict[int, set[int]] = {} + + def spare_slot_count(self, expert_count: int, eplb_slot_count: int) -> int: + """Return the spare slots per rank that hold the largest recoverable failure's experts.""" + lost_experts = min(self.largest_failure * eplb_slot_count, expert_count) + active_ranks = self.balancer.ep_size - self.largest_failure + return ceil_div(lost_experts, active_ranks) + + # EP rank r is global rank r because fault tolerance requires pipeline_parallel_size=1. + def _rank_nodes(self) -> list[int]: + return [self.balancer.mapping.get_node_rank(rank) for rank in range(self.balancer.ep_size)] + + @staticmethod + def _recoverable_failures(nodes: list[int]) -> list[list[int]]: + """Return the lost ranks of each recoverable failure. + + A recoverable failure loses a node other than rank 0's, or one rank on rank 0's node. + """ + node_ranks: dict[int, list[int]] = {} + for rank, node in enumerate(nodes): + node_ranks.setdefault(node, []).append(rank) + rank_0_node = node_ranks.pop(nodes[0], []) + return list(node_ranks.values()) + [[rank] for rank in rank_0_node[1:]] + + def plan_expert_restore( + self, + active_ranks: Sequence[bool], + current: Sequence[np.ndarray], + rejoining: Optional[Sequence[bool]] = None, + ) -> ExpertRestorePlan: + """Plan every layer's placement for ``active_ranks``. + + Args: + active_ranks: Whether each EP rank is active. + current: The per-layer placement that the active ranks agreed on, so that every + active rank plans the same placement. + rejoining: Whether each EP rank rejoins. If one does, the active slots of all active + ranks take the initial placement, which the rejoining rank loaded. + + Raises: + RuntimeError: An iteration is active. + ValueError: The active ranks cannot hold every expert, or a slot would need to load an + expert whose host weights no active rank's node keeps. + """ + balancer = self.balancer + active = np.asarray(active_ranks, dtype=bool) + if active.shape != (balancer.ep_size,) or not active[balancer.ep_rank]: + raise ValueError(f"EP rank {balancer.ep_rank} plans for active ranks {active.tolist()}") + if balancer.in_iter: + raise RuntimeError("Cannot plan an expert restore while an iteration is active") + rejoining_ranks = ( + np.zeros(balancer.ep_size, dtype=bool) + if rejoining is None + else np.asarray(rejoining, dtype=bool) & active + ) + digest = hashlib.sha256(active.tobytes()) + placements = [] + placements_before = [] + device_bytes = 0 + for i, (layer, layer_current) in enumerate( + zip(balancer.single_layer_load_balancers, current, strict=True) + ): + layer_current = np.asarray(layer_current, dtype=np.int32) + base = ( + layer.initial_placement.reshape(layer_current.shape) + if rejoining_ranks.any() + else layer_current + ) + placement = plan_layer_placement( + layer_current, + base, + active, + layer.active_slot_count_per_rank, + layer.expert_count, + i, + ) + # A rejoining rank loaded its initial-placement slots from storage. + before = np.where(rejoining_ranks[:, None], base, layer_current) + loaded = (placement >= 0) & (placement != before) + if loaded.any() and not layer.need_load_shared_weights(): + raise ValueError( + f"Layer {layer.get_layer_idx()} has no host weights to load experts " + f"{sorted(set(placement[loaded].tolist()))} from" + ) + device_bytes += spare_slot_bytes_to_back( + layer, filled_spare_slots(layer, placement[balancer.ep_rank]) + ) + digest.update(placement.tobytes()) + placements.append(placement) + placements_before.append(before) + return ExpertRestorePlan( + tuple(bool(rank) for rank in active), + tuple(placements), + tuple(placements_before), + digest.hexdigest()[:16], + device_bytes, + ) + + def set_spare_slots_backed(self, layer_slots: Sequence[Sequence[int]], backed: bool) -> int: + """Back spare slots with device memory or map them back to the shared scratch. + + Returns: + The number of bytes whose backing changed. + """ + tensors, slots, slot_counts = [], [], [] + for layer, spare_slots in zip( + self.balancer.single_layer_load_balancers, layer_slots, strict=True + ): + if not spare_slots: + continue + for tensor in layer.unbacked_slot_tensors.values(): + tensors.append(tensor) + slots.extend(spare_slots) + slot_counts.append(len(spare_slots)) + if not tensors: + return 0 + changed = torch.ops.trtllm.moe_unbacked_slots_set_backed( + tensors, slots, slot_counts, backed + ) + self.backed_spare_slot_bytes += changed if backed else -changed + return changed + + def release_empty_spare_slots(self) -> int: + """Map the spare slots that the installed placement leaves empty back to the shared scratch. + + An online EPLB update may still run, because it never fills these slots. + + Returns: + The number of bytes released. + """ + balancer = self.balancer + if balancer.in_iter: + raise RuntimeError("Cannot release spare slots while an iteration is active") + return self.set_spare_slots_backed( + [layer.get_unusable_local_slots() for layer in balancer.single_layer_load_balancers], + False, + ) + + def restore_lost_experts( + self, + plan: ExpertRestorePlan, + times: Optional[dict[str, float]] = None, + progress: Callable[[], None] = lambda: None, + ) -> int: + """Load the slots that ``plan`` changes on this rank, install it and return the bytes mapped. + + Every active rank must restore the same plan before any rank routes with it. A slot loads + from the host weights on this node. + + Args: + plan: The plan that every active rank restores. + times: Receives each phase's duration in milliseconds. + progress: Called after the backing and each layer loaded, so that a long restore keeps + reporting progress. It may raise to stop the restore. The slots loaded so far then + hold no expert in this rank's view until the next restore. + + Raises: + RuntimeError: An iteration is active, or the online EPLB worker may still write the + slots. + """ + balancer = self.balancer + if balancer.in_iter: + raise RuntimeError("Cannot restore experts while an iteration is active") + # No EPLB weight update may write the slots that the restore loads, or swap the placement + # that the check below reads. + balancer.load_balancer_impl.check_slots_writable() + layers = balancer.single_layer_load_balancers + if ( + not self._invalid_slots + and plan.active_ranks == self.active_ranks + and all( + np.array_equal(placement, layer.get_old_rank_expert_ids()) + for placement, layer in zip(plan.placements, layers) + ) + ): + return 0 + # The slots that the restore loads hold no expert in this rank's view until it installs. + for i in range(len(layers)): + loaded = np.flatnonzero(self._loaded_slots(plan, i)).tolist() + if loaded: + self._invalid_slots.setdefault(i, set()).update(loaded) + start_s = time.perf_counter() + # A spare slot is backed before an expert is loaded into it and before routing can reach it. + mapped = self.set_spare_slots_backed( + [ + filled_spare_slots(layer, placement[balancer.ep_rank]) + for layer, placement in zip(layers, plan.placements) + ], + True, + ) + start_s = _phase_done(times, "back", start_s) + progress() + self._load_local_slots(plan, progress) + start_s = _phase_done(times, "load", start_s) + balancer.load_balancer_impl.restore_placement( + [rank for rank, active in enumerate(plan.active_ranks) if not active], + [placement.tolist() for placement in plan.placements], + ) + _phase_done(times, "install", start_s) + self._invalid_slots.clear() + self.active_ranks = plan.active_ranks + return mapped + + def _loaded_slots(self, plan: ExpertRestorePlan, layer_idx: int) -> np.ndarray: + """Return whether ``plan`` loads each slot of this rank in layer ``layer_idx``.""" + ep_rank = self.balancer.ep_rank + placement = plan.placements[layer_idx][ep_rank] + return (placement >= 0) & (placement != plan.placement_before[layer_idx][ep_rank]) + + def _load_local_slots(self, plan: ExpertRestorePlan, progress: Callable[[], None]) -> None: + """Load this rank's slots that ``plan`` changes from this node's host weights.""" + ep_rank = self.balancer.ep_rank + for i, (layer, placement) in enumerate( + zip(self.balancer.single_layer_load_balancers, plan.placements) + ): + changed = self._loaded_slots(plan, i) + for slot in np.flatnonzero(changed): + expert = int(placement[ep_rank, slot]) + for name, tensor in layer.slot_tensors.items(): + host = layer.host_tensor_sharer.node_host_weights[(expert, name)] + _bytes(tensor[slot]).copy_(_bytes(host)) + progress() + + +class FaultTolerantMoeLoadBalancer(MoeLoadBalancer): + """MoE load balancer whose spare slots take the experts of failed ranks.""" + + fault_tolerance = True + + def __init__(self, *args, mapping: Mapping, **kwargs): + """Create the load balancer of one rank of a deployment. + + Args: + mapping: The mapping whose nodes decide which failures are recoverable. + """ + super().__init__(*args, **kwargs) + self.mapping = mapping + self.expert_restore = EplbExpertRestore(self) + self.host_tensor_sharer_cls = DeploymentHostWeights + + def _spare_slot_count(self, expert_count: int, slot_count_per_rank: int) -> int: + return self.expert_restore.spare_slot_count(expert_count, slot_count_per_rank) + + @property + def adds_spare_slots(self) -> bool: + # A layer gets spare slots exactly when some failure is recoverable. + return self.expert_restore.largest_failure > 0 + + def finalize_model(self): + super().finalize_model() + spare_slot_count = max( + (layer.spare_slot_count for layer in self.single_layer_load_balancers), default=0 + ) + if spare_slot_count and self.ep_rank == 0: + logger.info( + f"Wide-EP fault tolerance adds up to {spare_slot_count} unbacked spare slots per " + "rank" + ) + + def shutdown(self, local_safe: bool = True) -> bool: + """Release only this rank's resources. Other ranks on its node may have failed.""" + if self.is_shutdown: + return True + if not local_safe or self.in_iter: + return False + # The C++ shutdown waits for the weight updates in flight. Those read the host weights. + self.load_balancer_impl.shutdown() + for layer in self.single_layer_load_balancers: + layer.py_pre_shutdown_cleanup() + for layer in self.single_layer_load_balancers: + layer.py_post_shutdown_cleanup() + self.is_shutdown = True + return True + + def abandon_iterations(self) -> None: + """End the online EPLB work for the iterations that a rank failure interrupted. + + Every launched kernel must be able to finish, and no forward may launch until this returns. + """ + if self.in_iter: + raise RuntimeError("Cannot abandon iterations while an iteration is active") + self.load_balancer_impl.abandon_iterations() + for layer in self.single_layer_load_balancers: + layer.abandon_forward() diff --git a/tensorrt_llm/_torch/moe/fused_moe/impl_blocks.py b/tensorrt_llm/_torch/moe/fused_moe/impl_blocks.py index 5cd980d34c0c..79a8487bd827 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/impl_blocks.py +++ b/tensorrt_llm/_torch/moe/fused_moe/impl_blocks.py @@ -36,8 +36,9 @@ which is driven from the wrapper and stays on ``MoE``. """ +import functools import inspect -from typing import Dict, List, Optional, Union +from typing import Dict, List, Mapping, Optional, Union import torch @@ -171,7 +172,13 @@ def create_weights(self) -> None: if self._weights_created: return self.quant_method = self._get_quant_method() + # Only a module with the EPLB weight layout has a load balancer that gives it spare slots. + uses_spare_slots = isinstance(self, MoEEplbWeightLayoutMixin) and self.uses_spare_slots self.quant_method.create_weights(self) + if uses_spare_slots: + from .eplb_expert_restore import create_spare_slot_weights + + create_spare_slot_weights(self) self._weights_created = True self._check_configs() @@ -315,8 +322,26 @@ class MoEEplbWeightLayoutMixin: orchestration driven from the wrapper and stays on ``MoE``. """ + @functools.cached_property + def tuner_layout_kwargs(self) -> Mapping[str, int]: + """Return the layout of the local slots the MoE autotuner profiles, without spare slots. + + Fixed by the configuration; read after the load balancer exists. + """ + if not self.uses_spare_slots: + return {} + return {"tuner_num_local_experts": self.layer_load_balancer.active_slot_count_per_rank} + + @property + def uses_spare_slots(self) -> bool: + """Return whether the load balancer gives this module spare slots. + + Only an EPLB layer load balancer has a ``spare_slot_count``. + """ + return getattr(self.layer_load_balancer, "spare_slot_count", 0) > 0 + def _add_raw_shared_weights_for_unmap(self, weight_tensors: List[torch.Tensor]): - if self._using_dynamic_load_balancer(): + if self.layer_load_balancer and self.layer_load_balancer.need_load_shared_weights(): self.layer_load_balancer._add_raw_host_weight_for_unmap(weight_tensors) def _supports_load_balancer(self) -> bool: @@ -347,7 +372,7 @@ def register_parameter_weight_slot_fn(self, weight_name: str, local_slot_id: int def register_to_fix_weight_fn(self, weight_name: str): """Register weight fixing function for load balancer.""" - if not self._using_dynamic_load_balancer(): + if not (self._using_dynamic_load_balancer() or self.uses_spare_slots): return assert hasattr(self, weight_name), f"MoE doesn't have weight attr: {weight_name}" @@ -367,22 +392,28 @@ def register_to_fix_weight_fn(self, weight_name: str): f"numel={weight_tensor.numel()}, eltsize={weight_tensor.element_size()}, " f"dtype={weight_tensor.dtype}" ) - self.layer_load_balancer.make_tensor_host_accessible(weight_tensor) + if self.uses_spare_slots: + from .eplb_expert_restore import use_spare_slot_storage + + use_spare_slot_storage(self.layer_load_balancer, weight_tensor, weight_name) + else: + self.layer_load_balancer.make_tensor_host_accessible(weight_tensor) param.data = weight_tensor def register_all_parameter_slot_and_to_fix_weight_fns( self, weight_and_tensor_dict: Dict[str, torch.Tensor] ): """Register all parameter slot and weight fixing functions for load balancer.""" - if not self._using_dynamic_load_balancer(): + if not (self._using_dynamic_load_balancer() or self.uses_spare_slots): return - # Register weight functions for each local slot - for local_slot_id, expert_id in enumerate(self.initial_local_expert_ids): - for weight_name in weight_and_tensor_dict: - self.layer_load_balancer.add_register_weight_fn( - self.register_parameter_weight_slot_fn, (weight_name, local_slot_id) - ) + if self._using_dynamic_load_balancer(): + # Register weight functions for each local slot, spare slots included + for local_slot_id in range(self.expert_size_per_partition): + for weight_name in weight_and_tensor_dict: + self.layer_load_balancer.add_register_weight_fn( + self.register_parameter_weight_slot_fn, (weight_name, local_slot_id) + ) # Register weight migration functions for weight_name in weight_and_tensor_dict: @@ -400,6 +431,7 @@ def register_all_parameter_slot_and_to_fix_weight_fns( expert_id, weight_name, weight_tensor[local_slot_id] ) else: + # weight_tensor has no rows on a rank that loads no host weights. self.layer_load_balancer.host_tensor_sharer.pre_register_host_tensor_with_shape( - expert_id, weight_name, weight_tensor.dtype, weight_tensor[0].shape + expert_id, weight_name, weight_tensor.dtype, weight_tensor.shape[1:] ) diff --git a/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py b/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py index 756c884342e2..b32a43a3c4a3 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py +++ b/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py @@ -21,6 +21,8 @@ import torch +from tensorrt_llm._wide_ep_fault_tolerance import TURN_FAULT_TOLERANCE_OFF + if TYPE_CHECKING: from tensorrt_llm._torch.utils import ActivationType from tensorrt_llm.models.modeling_utils import QuantAlgo @@ -50,6 +52,10 @@ class MoEStaticCapability: # Whether the backend consumes a per-forward helper-slot plan whose live # weights may be published concurrently with shared-expert computation. supports_per_layer_dynamic_eplb: bool = False + # Whether the backend runs with the unbacked spare EPLB slots of Wide-EP + # fault tolerance: its autotuner profiles and its workspaces cover only the + # active slots. + supports_spare_slots: bool = False # Legacy gate: the ``assert moe_cls in [...]`` bias allow-list in # ``create_moe_backend``. Per-expert FC bias from the checkpoint, added # before the activation functor runs -- not an activation constant. @@ -316,6 +322,9 @@ class MoEDeployment: eplb_enabled: bool = False # Unified EPLB mode: disabled, standard, or per_layer_dynamic. eplb_mode: str = "disabled" + # True only when the registered load balancer adds unbacked spare slots + # (Wide-EP fault tolerance). + spare_slots_enabled: bool = False # True only for routed-expert LoRA targets. moe_lora_enabled: bool = False # False when ``moe_disable_finalize_fusion`` is set or any LoRA is @@ -363,6 +372,9 @@ class MoERejectReason(str, Enum): # EPLB is registered for this layer and the impl cannot lay out slots for # it. Distinct from TOPOLOGY_UNSUPPORTED: the parallel sizes are fine. EPLB_UNSUPPORTED = "eplb_unsupported" + # The load balancer adds the unbacked spare slots of Wide-EP fault + # tolerance, and the impl would autotune or allocate for them. + SPARE_SLOTS_UNSUPPORTED = "spare_slots_unsupported" # The deployment forces per-call activation scales # (``force_dynamic_quantization``) and the impl quantizes against scales # fixed at load time. Distinct from QUANT_UNSUPPORTED: the impl serves the @@ -490,6 +502,22 @@ def require_layer_quant_format(impl: "MoEImplBase", expected: str) -> None: ) +def spare_slots_rejection(cls: type, d: "MoEDeployment") -> Optional[MoEEligibility]: + """Reject the unbacked spare EPLB slots on an impl without ``supports_spare_slots``, else None. + + The resolver applies it to every candidate that ``can_implement`` accepts; a + family whose leaves differ also composes it into ``can_implement``, which a + direct call reaches. + """ + if d.spare_slots_enabled and not cls.capabilities.supports_spare_slots: + return MoEEligibility.no( + MoERejectReason.SPARE_SLOTS_UNSUPPORTED, + f"Wide-EP fault tolerance: {cls.__name__} does not support spare EPLB slots; " + f"select a MoE backend that does, or {TURN_FAULT_TOLERANCE_OFF}", + ) + return None + + def nvfp4_fc1_row_alignment_rejection( p: "MoEProblem", d: "MoEDeployment" ) -> Optional[MoEEligibility]: @@ -653,6 +681,7 @@ def to_dict(self) -> Dict[str, object]: "num_slots": self.deployment.num_slots, "eplb_enabled": self.deployment.eplb_enabled, "eplb_mode": self.deployment.eplb_mode, + "spare_slots_enabled": self.deployment.spare_slots_enabled, "moe_lora_enabled": self.deployment.moe_lora_enabled, "force_dynamic_quantization": self.deployment.force_dynamic_quantization, "oss_cutlass_moe": self.deployment.env.oss_cutlass_moe, diff --git a/tensorrt_llm/_torch/moe/fused_moe/interface.py b/tensorrt_llm/_torch/moe/fused_moe/interface.py index ce13b44c68c8..22af59c30bac 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/interface.py +++ b/tensorrt_llm/_torch/moe/fused_moe/interface.py @@ -598,6 +598,8 @@ def _init_load_balancer( top_k, slot_count_per_rank, aux_stream=aux_stream) + if not per_layer_dynamic_eplb: + self.expert_size_per_partition = self.layer_load_balancer.total_slot_count_per_rank self.repeat_count = self.layer_load_balancer.get_repeat_count() @@ -619,6 +621,10 @@ def _init_load_balancer( range(self.num_experts)) self.initial_global_assignments = loaded_initial_global_assignments + if not per_layer_dynamic_eplb: + self.initial_global_assignments = self.layer_load_balancer.with_spare_slots( + self.initial_global_assignments) + self.num_slots = len(self.initial_global_assignments) self.layer_load_balancer.set_initial_weight_assignments( self.initial_global_assignments) @@ -649,10 +655,13 @@ def _init_load_balancer( # mode. The layer manager separately owns compute slots M. self.slot_start = self.ep_rank * self.expert_size_per_partition self.slot_end = self.slot_start + self.expert_size_per_partition + # The weight loaders fill every local slot but the spare slots. + loaded_slot_count = self.expert_size_per_partition + if not per_layer_dynamic_eplb: + loaded_slot_count = self.layer_load_balancer.active_slot_count_per_rank self.initial_local_expert_ids = self.initial_global_assignments[ - self.slot_start:self.slot_end] - assert len( - self.initial_local_expert_ids) == self.expert_size_per_partition + self.slot_start:self.slot_start + loaded_slot_count] + assert len(self.initial_local_expert_ids) == loaded_slot_count else: # No EPLB: each rank gets a ceil/floor slice of num_experts. # Only backends that have validated their dispatch/combine paths diff --git a/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py b/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py index 2284ea235d81..cb49826615fb 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py +++ b/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py @@ -322,7 +322,11 @@ def __init__( expert_count: int, updates_enabled: bool = True, repeated_count=1, - aux_stream: Optional[torch.cuda.Stream] = None): + aux_stream: Optional[torch.cuda.Stream] = None, + *, + total_slot_count_per_rank: Optional[int] = None, + spare_slot_count: int = 0, + host_tensor_sharer_cls=HostMoeTensorSharer): """ Initialize a SingleLayerMoeLoadBalancer instance. @@ -332,6 +336,12 @@ def __init__( expert_count: total number of experts updates_enabled: whether to enable weight updates repeated_count: the repeated count of current layer, used when forward is repeated more than once like MTP. + total_slot_count_per_rank: the number of slots per rank, including + the spare slots. Required with spare slots. + spare_slot_count: the number of unbacked spare slots per rank for + Wide-EP fault tolerance. + host_tensor_sharer_cls: the class that shares the host weights of + the experts between the ranks of a node. """ self.single_layer_load_balancer_impl = single_layer_load_balancer_impl self.single_layer_load_balancer_ptr = single_layer_load_balancer_impl.get_pointer( @@ -339,10 +349,15 @@ def __init__( self.expert_count = expert_count self.updates_enabled = updates_enabled self.repeated_count = repeated_count + self.total_slot_count_per_rank = total_slot_count_per_rank + self.spare_slot_count = spare_slot_count + self.unbacked_slot_tensors: dict[str, torch.Tensor] = {} + self.slot_tensors: dict[str, torch.Tensor] = {} + self.initial_placement: Optional[np.ndarray] = None layer_id = self.single_layer_load_balancer_impl.get_layer_id() - self.host_tensor_sharer = HostMoeTensorSharer( + self.host_tensor_sharer = host_tensor_sharer_cls( layer_id, expert_count, - shared_mpi_comm) if self.updates_enabled else None + shared_mpi_comm) if self.need_load_shared_weights() else None self.register_weight_fns = [] self.to_migrate_weight_fns = [] @@ -390,13 +405,26 @@ def get_layer_idx(self): def get_old_rank_expert_ids(self): return self.single_layer_load_balancer_impl.get_old_rank_expert_ids() + def get_unusable_local_slots(self) -> list[int]: + return self.single_layer_load_balancer_impl.get_unusable_local_slots() + def get_load_expert_ids(self): - assert self.updates_enabled, "should not call get_load_expert_ids when using statistic routing" + assert self.need_load_shared_weights( + ), "should not call get_load_expert_ids when using statistic routing" return self.load_expert_ids def get_repeat_count(self): return self.repeated_count + @property + def active_slot_count_per_rank(self) -> int: + return self.total_slot_count_per_rank - self.spare_slot_count + + @property + def spare_slots(self) -> range: + return range(self.active_slot_count_per_rank, + self.total_slot_count_per_rank) + def is_static_routing(self): return not self.updates_enabled @@ -404,7 +432,7 @@ def is_dynamic_routing(self): return self.updates_enabled def need_load_shared_weights(self): - return self.is_dynamic_routing() + return self.is_dynamic_routing() or self.spare_slot_count > 0 def set_shared_memory_base_name(self, shared_memory_base_name): """ @@ -413,7 +441,7 @@ def set_shared_memory_base_name(self, shared_memory_base_name): Args: shared_memory_base_name: The base name for the shared memory """ - if self.updates_enabled: + if self.need_load_shared_weights(): self.host_tensor_sharer.set_shared_memory_base_name( shared_memory_base_name) @@ -475,14 +503,26 @@ def _add_host_weight_from_tensor(self, expert_id: int, name: str, host_weight = _tensor_to_weight(t) self._add_host_weight(expert_id, name, host_weight) + def with_spare_slots(self, eplb_assignments: list[int]) -> list[int]: + """Return ``eplb_assignments`` with each rank's spare slots appended empty.""" + if not self.spare_slot_count: + return eplb_assignments + per_rank = np.asarray(eplb_assignments).reshape( + -1, self.active_slot_count_per_rank) + spares = np.full((per_rank.shape[0], self.spare_slot_count), -1) + return np.concatenate([per_rank, spares], axis=1).ravel().tolist() + def set_initial_weight_assignments(self, initial_weight_assignments: List[int]): """ Set the initial weight assignments for the layer. Args: - initial_weight_assignments: A list of initial weight assignments + initial_weight_assignments: The expert of every slot, -1 for the spare slots. """ + if self.spare_slot_count: + self.initial_placement = np.array(initial_weight_assignments, + dtype=np.int32) self.single_layer_load_balancer_impl.set_initial_weight_assignments( initial_weight_assignments) @@ -514,6 +554,15 @@ def register_weight_slots_after_to_cuda(self): fn(*args, **kwargs) self.to_migrate_weight_fns = [] + without_host_weights = [ + name for name in sorted(self.unbacked_slot_tensors) + if name not in self.host_tensor_sharer.name_info + ] + if without_host_weights: + raise RuntimeError( + f"Weights {without_host_weights} of MoE layer " + f"{self.get_layer_idx()} have spare slots without device " + "memory but no host weights to restore an expert from") for fn, args, kwargs in self.register_weight_fns: fn(*args, **kwargs) @@ -525,9 +574,10 @@ def py_finalize_model(self): Finalize the model after all layers have been added. This must be called before starting any iterations. """ - if self.updates_enabled: + if self.need_load_shared_weights(): self.host_tensor_sharer.finalize_host_tensor_sharing( - self._add_host_weight_from_tensor) + self._add_host_weight_from_tensor if self. + updates_enabled else None) def start_wait_gpu_stage(self): """ @@ -589,6 +639,15 @@ def done_set_cpu_stage(self): if do_multi_stream(): self.event_dict[EventType.MoeBalancer].wait() + def abandon_forward(self): + """ + Reset the stage bookkeeping of a forward that did not complete, so + that the next forward starts afresh. + """ + for name in self.func_called_count: + self.func_called_count[name] = 0 + self.statistic_flag_tensor = None + def update_local_statistic(self, local_raw_expert_ids: torch.Tensor, is_first_stage: bool, is_last_stage: bool): """ @@ -764,14 +823,14 @@ def py_pre_shutdown_cleanup(self): Call after the C++ shutdown and before the barrier. """ - if self.updates_enabled: + if self.need_load_shared_weights(): self.host_tensor_sharer.pre_shutdown_cleanup() def py_post_shutdown_cleanup(self): """ Clean up the resources after C++ shutdown and barrier """ - if self.updates_enabled: + if self.need_load_shared_weights(): self.host_tensor_sharer.post_shutdown_cleanup() @@ -798,6 +857,11 @@ def is_static_routing(self) -> bool: def is_dynamic_routing(self) -> bool: return not self.is_static_routing() + @property + def adds_spare_slots(self) -> bool: + """Whether ``add_layer`` gives the layers unbacked spare slots (Wide-EP fault tolerance).""" + return False + @abstractmethod def set_repeated_for_next_layer(self, repeated_count: int) -> None: """Set how many times the next added layer is invoked per forward.""" @@ -891,6 +955,11 @@ class MoeLoadBalancer(BaseMoeLoadBalancer): This class can be used as a context manager to manage the lifecycle of a MoeLoadBalancer. """ + host_tensor_sharer_cls: Callable[..., + HostMoeTensorSharer] = HostMoeTensorSharer + # Whether the C++ load balancer can restore the placement after a rank failure. + fault_tolerance = False + def __init__( self, ep_rank: int, @@ -918,8 +987,11 @@ def __init__( self.ep_rank = ep_rank self.ep_size = ep_size self.layer_updates_per_iter = layer_updates_per_iter - self.load_balancer_impl = _tbr.MoeLoadBalancer(ep_rank, ep_size, - layer_updates_per_iter) + self.load_balancer_impl = _tbr.MoeLoadBalancer( + ep_rank, + ep_size, + layer_updates_per_iter, + fault_tolerance=self.fault_tolerance) self._previous_balancer = None self.single_layer_load_balancers = [] self.shared_memory_base_name = shared_memory_base_name or os.getenv( @@ -983,14 +1055,16 @@ def add_layer( Args: expert_count: The number of experts in the layer top_k: The number of experts each token selects - slot_count_per_rank: The number of slots per rank + slot_count_per_rank: The number of EPLB slots per rank, excluding spare slots. aux_stream: The auxiliary stream for overlapping Returns: A SingleLayerMoeLoadBalancer instance for the new layer """ + spare_slot_count = self._spare_slot_count(expert_count, + slot_count_per_rank) single_layer_load_balancer_impl = self.load_balancer_impl.add_layer( - expert_count, top_k, slot_count_per_rank) + expert_count, top_k, slot_count_per_rank + spare_slot_count) updates_enabled = not self.is_static_routing() repeat_count = 1 if self.next_layer_repeated_count is not None: @@ -1002,12 +1076,23 @@ def add_layer( expert_count, updates_enabled=updates_enabled, repeated_count=repeat_count, - aux_stream=aux_stream) + aux_stream=aux_stream, + total_slot_count_per_rank=slot_count_per_rank + spare_slot_count, + spare_slot_count=spare_slot_count, + host_tensor_sharer_cls=self.host_tensor_sharer_cls) single_layer_load_balancer.set_shared_memory_base_name( self.shared_memory_base_name) self.single_layer_load_balancers.append(single_layer_load_balancer) return single_layer_load_balancer + def _spare_slot_count(self, expert_count: int, + slot_count_per_rank: int) -> int: + """Return the number of spare slots per rank to add to a layer. + + A fault-tolerant load balancer overrides this. This one adds none. + """ + return 0 + def register_weight_slots_after_to_cuda(self): """ Register weights after model has been moved to cuda, should be invoked after model.to("cuda") and before finalize_model. @@ -1134,7 +1219,9 @@ def __repr__(self): def maybe_create_moe_load_balancer( - model_config, mapping: Optional[Mapping] + model_config, + mapping: Optional[Mapping], + load_balancer_cls: Callable[..., MoeLoadBalancer] = MoeLoadBalancer ) -> AbstractContextManager[BaseMoeLoadBalancer | None]: ep_rank = model_config.mapping.moe_ep_rank ep_size = model_config.mapping.moe_ep_size @@ -1203,7 +1290,7 @@ def maybe_create_moe_load_balancer( warmup_placement = _create_warmup_placement(model_config, ep_rank, ep_size) - moe_load_balancer = MoeLoadBalancer( + moe_load_balancer = load_balancer_cls( ep_rank=ep_rank, ep_size=ep_size, layer_updates_per_iter=config.layer_updates_per_iter, diff --git a/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py b/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py index 89779eb076e4..b5786aa0ceda 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py +++ b/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py @@ -57,6 +57,7 @@ canonical_routing, identity_quant_of, normalize_quant, + spare_slots_rejection, ) from .impl_environment import collect_moe_environment from .impl_identity import MOE_IMPL_REGISTRY, MoEImplDescriptor, MoEImplId, MoEImplQuery @@ -364,7 +365,8 @@ def build_moe_deployment( # MoE._init_load_balancer only adopts the config's slot count once a # balancer is registered; without one the layer keeps num_slots == # num_experts, so the deployment must say the same. - eplb_enabled = get_moe_load_balancer() is not None + balancer = get_moe_load_balancer() + eplb_enabled = balancer is not None balancer_config = getattr(model_config, "moe_load_balancer", None) eplb_mode = getattr(balancer_config, "mode", "standard") if eplb_enabled else "disabled" num_slots = getattr(balancer_config, "num_slots", None) if eplb_enabled else None @@ -384,6 +386,7 @@ def build_moe_deployment( # Registered balancer only; config alone does not enable EPLB. eplb_enabled=eplb_enabled, eplb_mode=eplb_mode, + spare_slots_enabled=eplb_enabled and balancer.adds_spare_slots, # Routed-expert LoRA only; attention-only LoRA stays False. moe_lora_enabled=has_moe_lora_targets(lora_config), force_dynamic_quantization=model_config.force_dynamic_quantization, @@ -1019,6 +1022,16 @@ def resolve_moe_impl( ) continue # After can_implement, so a backend's own more specific reason wins. + spare_slots = spare_slots_rejection(candidate, deployment) + if spare_slots is not None: + rejected.append( + MoERejection( + _legacy_backend_name(candidate), + spare_slots.reject_reason, + spare_slots.detail, + ) + ) + continue activation_rejection = _reject_unsupported_activation(candidate, problem) if activation_rejection is not None: rejected.append(activation_rejection) @@ -1055,6 +1068,17 @@ def resolve_moe_impl( f" Each candidate's reason: {report.describe_rejections()}" ) + # Spare slots are checked after the format, SM and dtype gates, so this + # rejection names a requested impl that serves the layer otherwise; with + # EPLB on, ``degraded_from`` may name another family member's EPLB gate. + spare_slots_cause = next( + ( + rejection + for rejection in report.requested_rejections + if rejection.reason is MoERejectReason.SPARE_SLOTS_UNSUPPORTED + ), + None, + ) if winner_cls is None: # Gates abstain rather than raise, so with all of them abstaining this # is the only place the individual reasons surface: ``describe()`` @@ -1065,6 +1089,19 @@ def resolve_moe_impl( f"(requested {requested}). Each candidate's reason: " f"{report.describe_rejections()}" ) + elif report.degraded and spare_slots_cause is not None: + # Every MoE layer of a fault-tolerant deployment falls back the same way, so the + # process logs it once. + logger.warning_once( + f"MoE backend {requested} cannot serve the MoE layers " + f"({spare_slots_cause.describe()}); running {winner_cls.__name__} instead.", + key=( + "moe_spare_slots_fallback", + requested, + winner_cls, + MoERejectReason.SPARE_SLOTS_UNSUPPORTED, + ), + ) elif report.degraded: cause = report.degraded_from others = ", ".join( diff --git a/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py index c4066021a85c..79d79604ca47 100644 --- a/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py +++ b/tensorrt_llm/_torch/pyexecutor/rank_failure_model_load.py @@ -23,6 +23,8 @@ import torch +from tensorrt_llm._wide_ep_fault_tolerance import TURN_FAULT_TOLERANCE_OFF + from ..moe.fused_moe.communication.execution_state import MoeExecutionState from ..moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided from ..moe.fused_moe.ep_group_health import EPGroupHealth @@ -32,9 +34,6 @@ if TYPE_CHECKING: from tensorrt_llm.mapping import Mapping -# The alternative that each error about an unmet fault tolerance requirement offers. -_TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" - def prepare_model(model: torch.nn.Module) -> None: """Bind every MoE layer of ``model`` to one execution state over all EP ranks. @@ -74,7 +73,7 @@ def moe_comms(model: torch.nn.Module) -> list[NVLinkOneSided]: f"MoE layer {name} ({type(layer).__name__}) uses {uses}, but Wide-EP fault " "tolerance requires NVLinkOneSided, the only MoE communication that can leave " f"out a failed rank. {_no_nvlink_one_sided_cause(layer.mapping)}; " - f"{_TURN_FAULT_TOLERANCE_OFF}." + f"{TURN_FAULT_TOLERANCE_OFF}." ) if first is None: first = name, comm @@ -84,12 +83,12 @@ def moe_comms(model: torch.nn.Module) -> list[NVLinkOneSided]: f"MoE layer {name} communicates as EP rank {comm.ep_rank} of {comm.ep_size}, but " f"MoE layer {first_name} as EP rank {first_comm.ep_rank} of {first_comm.ep_size}. " "Wide-EP fault tolerance runs every MoE layer over one EP group; give the layers " - f"the same MoE parallelism or {_TURN_FAULT_TOLERANCE_OFF}." + f"the same MoE parallelism or {TURN_FAULT_TOLERANCE_OFF}." ) comms[id(comm)] = comm if not comms: raise ValueError( - f"Wide-EP fault tolerance requires a model with MoE layers; {_TURN_FAULT_TOLERANCE_OFF}" + f"Wide-EP fault tolerance requires a model with MoE layers; {TURN_FAULT_TOLERANCE_OFF}" ) return list(comms.values()) diff --git a/tensorrt_llm/_wide_ep_fault_tolerance.py b/tensorrt_llm/_wide_ep_fault_tolerance.py new file mode 100644 index 000000000000..55c2135372b5 --- /dev/null +++ b/tensorrt_llm/_wide_ep_fault_tolerance.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Definitions that the modules of Wide-EP fault tolerance share.""" + +# The alternative that each error about an unmet fault tolerance requirement offers. +TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 08967afcef1a..0faa75ab1f6a 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -285,6 +285,8 @@ l0_b200: # ------------- MoE: test_moe_backend (by backend) --------------- - unittest/_torch/moe/test_moe_backend.py::test_cutlass_moe_routing_round_trip - unittest/_torch/moe/test_moe_backend.py::test_cutlass_moe_public_activation_ids + - unittest/_torch/moe/test_moe_backend.py::test_cutlass_moe_tunes_only_the_active_slots + - unittest/_torch/moe/test_moe_backend.py::test_cutedsl_nvfp4_grouped_gemm_tunes_only_the_active_slots - unittest/_torch/moe/test_moe_backend.py::test_trtllm_gen_mxfp4_fp8_input_quantization - unittest/_torch/moe/test_kimi_k3_moe_gate.py::test_fused_route_quant_matches_unfused_chain - unittest/_torch/moe/test_kimi_k3_moe_gate.py::test_router_logits_kernel_matches_router_gemm diff --git a/tests/unittest/_torch/moe/kernels/parallel/test_cute_dsl_moe.py b/tests/unittest/_torch/moe/kernels/parallel/test_cute_dsl_moe.py index be563becd30d..447dc0916956 100644 --- a/tests/unittest/_torch/moe/kernels/parallel/test_cute_dsl_moe.py +++ b/tests/unittest/_torch/moe/kernels/parallel/test_cute_dsl_moe.py @@ -246,6 +246,30 @@ def test_grouped_gemm_inputs_helper_fixed_expert_capacity(tile_size: int): helper.get_expert_capacity_num_permuted_tokens(0) +@pytest.mark.cpu_only +def test_inputs_helper_tunes_only_active_slots(): + """Tune rank 1's four active slots as without spare slots, and pad rows for all six slots.""" + num_tokens, tile_size = 256, 128 + helper = GroupedGemmInputsHelper( + num_experts=12, + top_k=4, + num_local_experts=6, + local_expert_offset=6, + tile_size=tile_size, + tuner_num_local_experts=4, + ) + without_spare_slots = GroupedGemmInputsHelper( + num_experts=8, top_k=4, num_local_experts=4, local_expert_offset=4, tile_size=tile_size + ) + + def tuning_routing(h: GroupedGemmInputsHelper) -> torch.Tensor: + num_tokens_per_expert = h.generate_num_tokens_per_expert(num_tokens, approx_max_load=True) + return h.generate_token_selected_experts(num_tokens, num_tokens_per_expert) + + assert torch.equal(tuning_routing(helper), tuning_routing(without_spare_slots)) + assert helper.get_expert_capacity_num_permuted_tokens(tile_size) == 6 * tile_size + + @pytest.mark.parametrize("tile_size", [128, 256]) @pytest.mark.parametrize("ep_size", [1, 8, 32]) @pytest.mark.parametrize("top_k", [1, 2, 8]) diff --git a/tests/unittest/_torch/moe/test_moe_backend.py b/tests/unittest/_torch/moe/test_moe_backend.py index 51b326a218ef..0aea705cfa30 100644 --- a/tests/unittest/_torch/moe/test_moe_backend.py +++ b/tests/unittest/_torch/moe/test_moe_backend.py @@ -350,6 +350,190 @@ def run_moe(): torch.testing.assert_close(captured, torch.stack(expected), atol=0.01, rtol=0.05) +def test_cutlass_moe_tunes_only_the_active_slots(monkeypatch: pytest.MonkeyPatch) -> None: + """Tune only the active slots and reuse the tactics once restored experts fill spare slots.""" + if not torch.cuda.is_available() or get_sm_version() < 80: + pytest.skip("BF16 CUTLASS MoE requires CUDA and SM80 or newer") + from tensorrt_llm._torch.custom_ops.torch_custom_ops import MoERunner + + torch.manual_seed(17) + dtype = torch.bfloat16 + num_active, num_spare, hidden_size, inter_size, num_tokens = 4, 2, 128, 128, 64 + num_slots = num_active + num_spare + x = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) * 0.5 + w1 = torch.randn(num_slots, 2 * inter_size, hidden_size, device="cuda", dtype=dtype) * 0.1 + w2 = torch.randn(num_slots, hidden_size, inter_size, device="cuda", dtype=dtype) * 0.1 + tokens = torch.arange(num_tokens, device="cuda", dtype=torch.int32) + selected = tokens.remainder(num_active).unsqueeze(1) + # Experts restored into the spare slots receive tokens like the others. + restored = tokens.remainder(num_slots).unsqueeze(1) + scores = torch.ones(num_tokens, 1, device="cuda", dtype=torch.float32) + + def run_moe( + fc1: torch.Tensor, fc2: torch.Tensor, routed: torch.Tensor = selected, **kwargs + ) -> torch.Tensor: + return torch.ops.trtllm.fused_moe( + x, routed, scores, fc1, None, fc2, None, dtype, [], **kwargs + )[0] + + active_slots = dict(tuner_num_local_experts=num_active) + # The references run before tuning, with the fallback tactics. + expected = run_moe(w1[:num_active], w2[:num_active]) + expected_restored = run_moe(w1, w2, restored) + # The kernel runner the op uses for BF16, recording the experts of the weights it profiles. + instance_key = (dtype, dtype, dtype, False, False, False, False, False) + kernel_runner = MoERunner.runner_dict[instance_key] + profiled = [] + + class _RecordingRunner: + def run_gemm_profile(self, *args): + profiled.append((args[1].size(0), args[3].size(0))) + return kernel_runner.run_gemm_profile(*args) + + def __getattr__(self, name: str): + return getattr(kernel_runner, name) + + monkeypatch.setitem(MoERunner.runner_dict, instance_key, _RecordingRunner()) + tuner = AutoTuner.get() + tuner.clear_cache() + with autotune(): + run_moe(w1, w2, **active_slots) + assert profiled + assert set(profiled) == {(num_active, num_active)} + + hits = [] + search_cache = tuner.profiling_cache.search_cache + + def recording_search_cache(*args, **kwargs): + result = search_cache(*args, **kwargs) + hits.append(result[0]) + return result + + monkeypatch.setattr(tuner.profiling_cache, "search_cache", recording_search_cache) + for routed, reference in ((selected, expected), (restored, expected_restored)): + hits.clear() + actual = run_moe(w1, w2, routed, **active_slots) + assert hits == [True, True] + torch.testing.assert_close(actual, reference, atol=1e-2, rtol=1e-2) + + +def test_cutedsl_nvfp4_grouped_gemm_tunes_only_the_active_slots( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Tune only the active slots and reuse the tactics once restored experts fill spare slots.""" + from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE + + if not torch.cuda.is_available() or get_sm_version() not in (100, 103): + pytest.skip("The CuteDSL NVFP4 grouped GEMM requires SM100 or SM103") + if not IS_CUTLASS_DSL_AVAILABLE: + pytest.skip("CuteDSL is not available") + from tensorrt_llm._torch.custom_ops import cute_dsl_custom_ops + + torch.manual_seed(17) + sf_vec_size, tile_size = 16, 128 + num_active, num_spare, hidden_size, inter_size, num_tokens = 4, 2, 512, 512, 256 + num_slots = num_active + num_spare + w = torch.randint( + -5, 5, (num_slots, inter_size, hidden_size), dtype=torch.bfloat16, device="cuda" + ) + w_global_sf = w.abs().amax(dim=(1, 2)).float() / (448 * 6) + w, w_sf = torch.ops.trtllm.fp4_quantize(w, 1 / w_global_sf, sf_vec_size, False) + w = w.view(torch.float4_e2m1fn_x2) + w_sf = w_sf.view(num_slots, inter_size, hidden_size // sf_vec_size) + + def gemm_inputs(routed: torch.Tensor) -> Tuple[list, int]: + """Return the grouped GEMM inputs of ``routed`` tokens and the rows of their tiles.""" + tile_idx_to_group_idx, *_, num_non_exiting_tiles = torch.ops.trtllm.moe_sort( + token_selected_experts=routed, + token_final_scales=torch.ones(routed.shape, dtype=torch.float32, device="cuda"), + num_experts=num_slots, + top_k=1, + local_expert_offset=0, + local_num_experts=num_slots, + tile_tokens_dim=tile_size, + ) + x = torch.randint( + -5, + 5, + (tile_idx_to_group_idx.numel() * tile_size, hidden_size), + dtype=torch.bfloat16, + device="cuda", + ) + x_global_sf = x.abs().max().float() / (448 * 6) + x, x_sf = torch.ops.trtllm.fp4_quantize(x, 1 / x_global_sf, sf_vec_size, False) + inputs = [ + x.view(torch.float4_e2m1fn_x2), + w, + x_sf, + w_sf, + x_global_sf * w_global_sf, + tile_idx_to_group_idx, + num_non_exiting_tiles, + ] + return inputs, num_non_exiting_tiles.item() * tile_size + + def run_gemm(inputs: list, **kwargs) -> torch.Tensor: + return torch.ops.trtllm.cute_dsl_nvfp4_grouped_gemm_blackwell( + *inputs, + num_experts=num_slots, + top_k=1, + num_local_experts=num_slots, + local_expert_offset=0, + tile_size=tile_size, + output_dtype=torch.bfloat16, + scaling_vector_size=sf_vec_size, + **kwargs, + ) + + tokens = torch.arange(num_tokens, device="cuda", dtype=torch.int32) + selected, selected_rows = gemm_inputs(tokens.remainder(num_active).unsqueeze(1)) + # Experts restored into the spare slots receive tokens like the others. + restored, restored_rows = gemm_inputs(tokens.remainder(num_slots).unsqueeze(1)) + active_slots = dict(tuner_num_local_experts=num_active) + # The references run before tuning, with the fallback tactics. + expected = run_gemm(selected) + expected_restored = run_gemm(restored) + # The experts of the routing that the tuner generates for each profile. + profiled = set() + inputs_pre_hook = cute_dsl_custom_ops.GroupedGemmInputsHelper.inputs_pre_hook + + def recording_inputs_pre_hook(helper, inputs): + tensors = inputs_pre_hook(helper, inputs) + tile_idx_to_group_idx, num_non_exiting_tiles = tensors[5], tensors[6] + profiled.update(tile_idx_to_group_idx[: num_non_exiting_tiles.item()].tolist()) + return tensors + + monkeypatch.setattr( + cute_dsl_custom_ops.GroupedGemmInputsHelper, "inputs_pre_hook", recording_inputs_pre_hook + ) + monkeypatch.setattr( + cute_dsl_custom_ops.Sm100BlockScaledContiguousGroupedGemmRunner, "tuning_config_cache", {} + ) + tuner = AutoTuner.get() + tuner.clear_cache() + with autotune(): + run_gemm(selected, **active_slots) + assert profiled == set(range(num_active)) + + hits = [] + search_cache = tuner.profiling_cache.search_cache + + def recording_search_cache(*args, **kwargs): + result = search_cache(*args, **kwargs) + hits.append(result[0]) + return result + + monkeypatch.setattr(tuner.profiling_cache, "search_cache", recording_search_cache) + for inputs, rows, reference in ( + (selected, selected_rows, expected), + (restored, restored_rows, expected_restored), + ): + hits.clear() + actual = run_gemm(inputs, **active_slots) + assert hits == [True] + torch.testing.assert_close(actual[:rows], reference[:rows], atol=1e-2, rtol=1e-2) + + _MEGAMOE_BACKEND_TYPES = { MoeBackendType.MEGAMOE_DEEPGEMM, MoeBackendType.MEGAMOE_CUTEDSL, @@ -620,6 +804,46 @@ def test_cutedsl_direct_metadata_tuning_buckets_capacity_and_legacy_uses_tokens( assert legacy_dynamic_spec.map_to_tuning_buckets(expert_capacity_rows) == 512 +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "runner", + [ + "Sm100BlockScaledContiguousGroupedGemmRunner", + "Sm100BlockScaledContiguousGroupedGemmFinalizeFusionRunner", + "Sm100BlockScaledContiguousGatherGroupedGemmActFusionRunner", + "CuteDslFusedMoENvfp4Runner", + ], +) +def test_cutedsl_runner_identity_has_tuner_layout( + monkeypatch: pytest.MonkeyPatch, runner: str +) -> None: + """A tuning of only the active slots is cached apart from a tuning of every local slot.""" + from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE + + if not IS_CUTLASS_DSL_AVAILABLE: + pytest.skip("CuteDSL is not available") + from tensorrt_llm._torch.custom_ops import cute_dsl_custom_ops + + monkeypatch.setattr(cute_dsl_custom_ops, "get_sm_version", lambda: 100) + layout = {"num_experts": 12, "top_k": 4, "num_local_experts": 6, "local_expert_offset": 6} + + def build(tuner_num_local_experts: int | None): + if runner == "CuteDslFusedMoENvfp4Runner": + return CuteDslFusedMoENvfp4Runner( + forward_impl=MagicMock(), + **layout, + tuner_num_local_experts=tuner_num_local_experts, + ) + runner_cls = getattr(cute_dsl_custom_ops, runner) + extra = {} if "ActFusion" in runner else {"output_dtype": torch.bfloat16} + return runner_cls( + **layout, tile_size=128, **extra, tuner_num_local_experts=tuner_num_local_experts + ) + + every_slot = build(None).unique_id() + assert build(4).unique_id() == every_slot + ("tuner_num_local_experts", 4) + + @pytest.mark.cpu_only @pytest.mark.parametrize("decode_tuning", [False, True]) def test_fp4_block_scale_moe_decode_tuning(decode_tuning: bool) -> None: @@ -980,6 +1204,124 @@ def test_cutedsl_rejects_direct_metadata_without_fused_finalize() -> None: assert backend.can_use_deep_ep_direct_metadata(True) is False +def _cutedsl_nvfp4_after_create_weights(spare_slot_count: int, monkeypatch): + """A CuteDSL NVFP4 backend whose create_weights ran with ``spare_slot_count`` spare slots and + 3 active slots per rank. Without spare slots, reading the tuner layout fails the test.""" + from tensorrt_llm._torch.moe.fused_moe.cutedsl.grouped_gemm import nvfp4 + + monkeypatch.setattr(nvfp4, "get_sm_version", lambda: 100) + if not spare_slot_count: + + def tuner_layout_kwargs(self): + pytest.fail("a module without spare slots read the tuner layout") + + monkeypatch.setattr( + TrtllmCutedslGroupedGemmNvfp4Impl, "tuner_layout_kwargs", property(tuner_layout_kwargs) + ) + backend = TrtllmCutedslGroupedGemmNvfp4Impl.__new__(TrtllmCutedslGroupedGemmNvfp4Impl) + backend._weights_created = False + backend._check_quant_config_is_my_format = lambda: None + backend.layer_load_balancer = SimpleNamespace( + spare_slot_count=spare_slot_count, active_slot_count_per_rank=3 + ) + monkeypatch.setattr(MoEWeightOwnerMixin, "create_weights", lambda self: None) + backend.create_weights() + backend._locality_domain_runtime = None + backend._pending_x_ready_event = None + backend.hidden_size = 4 + backend.num_slots = 4 + backend.routing_method = None + return backend + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("spare_slot_count", [0, 2]) +def test_cutedsl_nvfp4_runner_gets_the_tuner_layout_only_with_spare_slots( + spare_slot_count, monkeypatch +): + """The forward builds its runner with the active slots' tuner layout only with spare slots.""" + from tensorrt_llm._torch.moe.fused_moe.cutedsl.grouped_gemm import nvfp4 + + runners = [] + + class Runner: + def __init__(self, **kwargs): + runners.append(kwargs) + + def get_tuning_config(self): + return None + + def __call__(self, inputs, tactic): + return "output" + + monkeypatch.setattr(nvfp4, "CuteDslFusedMoENvfp4Runner", Runner) + monkeypatch.setattr( + nvfp4, + "AutoTuner", + SimpleNamespace(get=lambda: Mock(**{"choose_one.return_value": (None, 0)})), + ) + backend = _cutedsl_nvfp4_after_create_weights(spare_slot_count, monkeypatch) + backend.use_fused_finalize = True + + output = backend.run_moe_nvfp4( + MagicMock(), + torch.zeros((2, 1), dtype=torch.int32), + torch.ones((2, 1), dtype=torch.float32), + moe_output=torch.empty((2, 4), dtype=torch.bfloat16), + weight_view=MagicMock(expert_size_per_partition=4, slot_start=0), + ) + + assert output == "output" and len(runners) == 1 + assert runners[0].get("tuner_num_local_experts") == (3 if spare_slot_count else None) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("use_fused_finalize", [True, False]) +@pytest.mark.parametrize("spare_slot_count", [0, 2]) +def test_cutedsl_nvfp4_grouped_gemms_get_the_tuner_layout_only_with_spare_slots( + spare_slot_count, use_fused_finalize, monkeypatch +): + """The forward's grouped GEMMs get the active slots' tuner layout only with spare slots.""" + calls = {} + + def record(name, result=None): + def op(**kwargs): + calls[name] = kwargs + return result + + monkeypatch.setattr(torch.ops.trtllm, name, op, raising=False) + + gather = "cute_dsl_nvfp4_gather_grouped_gemm_act_fusion_blackwell" + finalize = "cute_dsl_nvfp4_grouped_gemm_finalize_inplace_blackwell" + gemm = "cute_dsl_nvfp4_grouped_gemm_blackwell" + record(gather, (MagicMock(), MagicMock())) + record(finalize) + record(gemm, MagicMock()) + record("moe_output_memset_inplace") + record("moe_unpermute_inplace") + backend = _cutedsl_nvfp4_after_create_weights(spare_slot_count, monkeypatch) + backend.use_fused_finalize = use_fused_finalize + backend._has_moe_output_memset_aux_stream = lambda: False + backend.mapping = SimpleNamespace(moe_ep_size=1) + backend.fc2_input_scale = None + backend.activation_type = ActivationType.Swiglu + backend.act_clamp = None + sort_tables = SimpleNamespace(tile_size=128, outputs=tuple(MagicMock() for _ in range(6))) + + backend.run_moe_nvfp4_impl( + MagicMock(), + torch.zeros((2, 1), dtype=torch.int32), + torch.ones((2, 1), dtype=torch.float32), + MagicMock(), + MagicMock(), + MagicMock(expert_size_per_partition=4, slot_start=0), + sort_tables=sort_tables, + ) + + for name in (gather, finalize if use_fused_finalize else gemm): + assert calls[name].get("tuner_num_local_experts") == (3 if spare_slot_count else None) + + @pytest.mark.cpu_only def test_cutedsl_direct_metadata_request_fails_without_fused_finalize() -> None: backend = TrtllmCutedslGroupedGemmNvfp4Impl.__new__(TrtllmCutedslGroupedGemmNvfp4Impl) @@ -3670,7 +4012,8 @@ def test_post_silu_clamp_mode_requires_limit(): def test_fused_moe_appends_post_silu_mode_to_positional_schema(): registered_schema = torch.ops.trtllm.fused_moe.default._schema - assert registered_schema.arguments[-1].name == "swiglu_clamp_after_silu" + assert registered_schema.arguments[-2].name == "swiglu_clamp_after_silu" + assert registered_schema.arguments[-1].name == "tuner_num_local_experts" @pytest.mark.cpu_only @@ -7164,6 +7507,7 @@ def _cutlass_leaf_on_cpu(leaf_cls): leaf.hidden_size = 8 leaf.unpadded_hidden_size = 8 leaf.force_dynamic_quantization = False + leaf.layer_load_balancer = None # Stubbed on every leaf, so only whether run_moe calls it decides. leaf._extract_moe_lora_tensors = lambda lora_params: dict(_LORA_SENTINEL) return leaf @@ -7227,6 +7571,38 @@ def test_cutlass_leaf_run_moe_hands_the_op_its_declarations(leaf_cls, monkeypatc assert (calls[0].get("fc1_lora_ranks") == "fc1-ranks") is fuses_lora +@pytest.mark.cpu_only +@pytest.mark.parametrize("leaf_cls", _CUTLASS_FUSED_MOE_OP_LEAVES, ids=lambda cls: cls.__name__) +def test_cutlass_leaf_without_spare_slots_never_reads_the_tuner_layout(leaf_cls, monkeypatch): + """Without spare slots the op runs with its own arguments only, and no tuner layout is read per call.""" + calls = _record_fused_moe(monkeypatch) + + def tuner_layout_kwargs(self): + pytest.fail("a forward without spare slots read the tuner layout") + + monkeypatch.setattr(leaf_cls, "tuner_layout_kwargs", property(tuner_layout_kwargs)) + leaf = _cutlass_leaf_on_cpu(leaf_cls) + leaf.run_moe(_cpu_run_context()) + + assert len(calls) == 1 and "tuner_num_local_experts" not in calls[0] + assert leaf._fused_moe_op is torch.ops.trtllm.fused_moe + + +@pytest.mark.cpu_only +def test_cutlass_leaf_with_spare_slots_tunes_its_active_slots(monkeypatch): + """A layer with spare slots binds the tuner layout of its active slots once, at weight creation.""" + calls = _record_fused_moe(monkeypatch) + leaf = _cutlass_leaf_on_cpu(_CUTLASS_FUSED_MOE_OP_LEAVES[0]) + leaf.layer_load_balancer = SimpleNamespace(spare_slot_count=2, active_slot_count_per_rank=3) + leaf._weights_created = False + leaf._check_quant_config_is_my_format = lambda: None + monkeypatch.setattr(MoEWeightOwnerMixin, "create_weights", lambda self: None) + leaf.create_weights() + leaf.run_moe(_cpu_run_context()) + + assert len(calls) == 1 and calls[0]["tuner_num_local_experts"] == 3 + + @pytest.mark.cpu_only def test_cutlass_w4a16_nvfp4_runs_the_op_unquantized_on_dequantized_weights(monkeypatch): calls = _record_fused_moe(monkeypatch) diff --git a/tests/unittest/_torch/moe/test_moe_impl.py b/tests/unittest/_torch/moe/test_moe_impl.py index e8e395db442c..82e4e627d187 100644 --- a/tests/unittest/_torch/moe/test_moe_impl.py +++ b/tests/unittest/_torch/moe/test_moe_impl.py @@ -464,6 +464,87 @@ def test_per_layer_dynamic_eplb_filters_backends_by_declared_capability(): assert [item.reason for item in rejected.rejected] == [MoERejectReason.EPLB_UNSUPPORTED] +def test_spare_slots_filter_backends_by_declared_capability(): + """With spare EPLB slots, selection turns down a backend that does not declare them.""" + deployment = MoEDeployment( + ep_size=1, + tp_size=1, + parallel_size=1, + use_dp=False, + num_slots=8, + env=_deepgemm_environment(), + eplb_enabled=True, + eplb_mode="standard", + ) + problem = MoEProblem( + quant=canonical_quant(QuantAlgo.FP8_BLOCK_SCALES), dtype_act=torch.bfloat16 + ) + accepted = resolve_moe_impl( + _deepgemm_model_config(), + problem=problem, + deployment=deployment, + impl_id=_DEEPGEMM_IMPL_ID, + ) + assert impl_class_for(accepted) is DeepgemmCudaFp8BlockScalesImpl + + rejected = resolve_moe_impl( + _deepgemm_model_config(), + problem=problem, + deployment=dataclasses.replace(deployment, spare_slots_enabled=True), + impl_id=_DEEPGEMM_IMPL_ID, + ) + assert rejected.winner is None + assert [item.reason for item in rejected.rejected] == [MoERejectReason.SPARE_SLOTS_UNSUPPORTED] + assert "enable_wide_ep_fault_tolerance=False" in rejected.rejected[0].detail + + +def test_spare_slots_fall_back_from_a_backend_literal_and_fail_a_pin(monkeypatch): + """A backend literal whose leaf lacks spare EPLB slots falls back with one warning for every + layer, which names fault tolerance and the reason; a pin of that leaf fails.""" + logger = moe_resolution.logger + log = MagicMock() + monkeypatch.setattr(logger, "log", log) + monkeypatch.setattr(logger, "_appeared_keys", set()) + config = _cutedsl_model_config(None) + deployment = dataclasses.replace( + _single_rank_deployment(_cutedsl_environment()), + eplb_enabled=True, + eplb_mode="standard", + spare_slots_enabled=True, + ) + for layer_idx in (3, 4): + by_literal = resolve_moe_impl(config, deployment=deployment, layer_idx=layer_idx) + assert impl_class_for(by_literal) is TrtllmCutlassUnquantizedImpl + # EPLB turns down the B12x leaf first, before its format gate. + assert { + rejection.legacy_backend: rejection.reason + for rejection in by_literal.requested_rejections + }[TrtllmCutedslGroupedGemmBf16Impl.__name__] is MoERejectReason.SPARE_SLOTS_UNSUPPORTED + + # One line for the process, the same for every layer. + [warning] = [call.args[1] for call in log.call_args_list if call.args[0] == logger.WARNING] + assert ( + "Wide-EP fault tolerance: TrtllmCutedslGroupedGemmBf16Impl does not support spare EPLB " + "slots" in warning + ) + assert f"running {TrtllmCutlassUnquantizedImpl.__name__} instead" in warning + + pinned = resolve_moe_impl(config, deployment=deployment, impl_id=_CUTEDSL_GROUPED_GEMM_IDS[1]) + with pytest.raises(ValueError, match="Wide-EP fault tolerance: TrtllmCutedslGroupedGemmBf16"): + impl_class_for(pinned) + + +@pytest.mark.parametrize("adds_spare_slots", [False, True]) +def test_deployment_takes_spare_slots_from_the_load_balancer(monkeypatch, adds_spare_slots): + balancer = SimpleNamespace(adds_spare_slots=adds_spare_slots) + monkeypatch.setattr(moe_resolution, "get_moe_load_balancer", lambda: balancer) + deployment = moe_resolution.build_moe_deployment( + ModelConfig(), num_experts=8, environment=MoEEnvironment(sm=100) + ) + assert deployment.eplb_enabled + assert deployment.spare_slots_enabled is adds_spare_slots + + def test_pinned_megamoe_cutedsl_identity_fails_hard_on_another_format(): """A pin that cannot run must fail, not widen to the sibling that can. @@ -2202,6 +2283,7 @@ def _cutedsl_model_config(quant_algo: QuantAlgo | None = QuantAlgo.NVFP4) -> Mod supports_eplb=True, supports_apply_router_weight_on_input=True, supports_deep_ep_direct_metadata=True, + supports_spare_slots=True, ), MoEActivationSupport( kinds=frozenset({ActivationType.Swiglu, ActivationType.Relu2, ActivationType.SiTu}), diff --git a/tests/unittest/_torch/moe/test_moe_load_balancer.py b/tests/unittest/_torch/moe/test_moe_load_balancer.py index e971ad0fe6e2..f89621f3931d 100644 --- a/tests/unittest/_torch/moe/test_moe_load_balancer.py +++ b/tests/unittest/_torch/moe/test_moe_load_balancer.py @@ -1,9 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import functools +import gc import os import unittest from collections.abc import Iterator +from contextlib import nullcontext from unittest.mock import MagicMock, patch import pytest @@ -12,6 +15,11 @@ from transformers import PretrainedConfig from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.moe.fused_moe.eplb_expert_restore import ( + EplbExpertRestore, FaultTolerantMoeLoadBalancer, create_spare_slot_storage, + spare_slot_bytes_to_back, use_spare_slot_storage) +from tensorrt_llm._torch.moe.fused_moe.impl_blocks import ( + MoEEplbWeightLayoutMixin, MoEWeightOwnerMixin) from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import ( MoeLoadBalancer, MoeLoadBalancerIterContext, SingleLayerMoeLoadBalancer, get_moe_load_balancer, maybe_create_moe_load_balancer, @@ -44,8 +52,10 @@ def test_moe_load_balancer_init(self, mock_load_balancer_impl): balancer = MoeLoadBalancer(ep_rank, ep_size, layer_updates_per_iter) # Verify - mock_load_balancer_impl.assert_called_once_with(ep_rank, ep_size, - layer_updates_per_iter) + mock_load_balancer_impl.assert_called_once_with(ep_rank, + ep_size, + layer_updates_per_iter, + fault_tolerance=False) self.assertEqual(balancer.ep_rank, ep_rank) self.assertEqual(balancer.ep_size, ep_size) self.assertEqual(balancer.layer_updates_per_iter, @@ -166,7 +176,10 @@ def test_kimi_k25_creates_moe_load_balancer(self, mock_load_balancer_impl): 104) self.assertEqual(get_moe_load_balancer(), balancer) - mock_load_balancer_impl.assert_called_once_with(0, 4, 2) + mock_load_balancer_impl.assert_called_once_with(0, + 4, + 2, + fault_tolerance=False) @patch('tensorrt_llm.bindings.internal.runtime.MoeLoadBalancer') def test_exception_in_context(self, mock_load_balancer_impl): @@ -604,5 +617,438 @@ def test_shutdown_stops_cpp_before_closing_host_weights( assert order == ["C++ shutdown", "host weights closed"] +class _WeightLayout(MoEWeightOwnerMixin, MoEEplbWeightLayoutMixin, + torch.nn.Module): + """A MoE weight owner whose quantization method creates no weights.""" + + def _get_quant_method(self): + return MagicMock() + + +def _skip_without_gdrcopy_slots(): + """Skip unless online EPLB's slots are GDRCopy mappings.""" + from tensorrt_llm.bindings.internal import runtime + + # A load balancer initializes host accessible device memory. + balancer = runtime.MoeLoadBalancer(0, 1, 0) + uses_gdrcopy = runtime.host_accessible_device_memory_uses_gdrcopy() + balancer.shutdown() + del balancer + if not uses_gdrcopy: + pytest.skip("host accessible EPLB slots are not GDRCopy mappings here") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +@pytest.mark.parametrize("layer_updates_per_iter", [0, 1], + ids=["static", "online"]) +def test_lost_expert_restored_into_spare_slot(monkeypatch, + layer_updates_per_iter): + """Restore rank 1's expert 3 into rank 0's spare slot, then release it on rejoin.""" + if layer_updates_per_iter: + _skip_without_gdrcopy_slots() + monkeypatch.setenv("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1") + # Two ranks on two nodes. Rank 0 gets three spare slots after EPLB's + # three, since rank 1's slots may hold three experts rank 0 does not. + monkeypatch.setattr(EplbExpertRestore, "_rank_nodes", lambda self: [0, 1]) + torch.cuda.set_device(0) + expert_count = 4 + # Slots of 8 MiB (whole VMM chunks), of 3 MiB + 4 bytes (chunks shared + # with the neighbor slots) and of 4 bytes (a per-slot scale, whose spare + # slot fills no whole chunk, so it keeps its storage without spare slots). + slot_elements = {"w": 1 << 21, "s": (3 << 18) + 1, "alpha": 1} + mib = 1 << 20 + # The one VMM chunk all unbacked pieces of the process map: released once no piece maps it. + scratch = 2 * mib + balancer = FaultTolerantMoeLoadBalancer( + 0, + 2, + layer_updates_per_iter, + shared_memory_base_name=f"test_restore_{os.getpid()}", + mapping=None) + try: + # Top-1: route() selects one expert per token. + layer = balancer.add_layer(expert_count, 1, 3) + assert layer.total_slot_count_per_rank == 6 + assert layer.spare_slot_count == 3 + slot_count = layer.total_slot_count_per_rank + assignments = layer.with_spare_slots([0, 1, 2, 3, 0, 1]) + local_experts = assignments[:slot_count] + host = { + name: [torch.randn(elements) for _ in range(expert_count)] + for name, elements in slot_elements.items() + } + weights = {} + active = layer.active_slot_count_per_rank + for name, elements in slot_elements.items(): + # As loaded: the active slots hold their experts, the spare slots nothing. + loaded = torch.stack( + [host[name][e] for e in local_experts[:active]] + + [torch.zeros(elements)] * (slot_count - active)) + storage = create_spare_slot_storage(layer, name, loaded) + weights[name] = loaded.cuda() if storage is None else storage + for name in slot_elements: + for expert_id in range(expert_count): + layer.host_tensor_sharer.share_host_tensor_with_shape( + expert_id, name, host[name][expert_id]) + layer.add_to_migrate_weight_fn(use_spare_slot_storage, + (layer, weights[name], name)) + for slot in range(layer.total_slot_count_per_rank): + if layer_updates_per_iter: + layer.add_register_weight_fn( + lambda slot=slot, name=name: layer.register_weight_slot( + slot, name, weights[name][slot]), ()) + layer.host_tensor_sharer.finalize_layer_weights() + layer.set_initial_weight_assignments(assignments) + + def weight_owner(): + # A MoE layer's weight owner, whose MoE ops take the tuner layout + # of its layer's spare slots. + owner = _WeightLayout() + owner.layer_load_balancer, owner.num_experts = layer, expert_count + owner.expert_size_per_partition = layer.total_slot_count_per_rank + owner.initial_local_expert_ids = local_experts + owner.create_weights() + owner.register_all_parameter_slot_and_to_fix_weight_fns({}) + return owner + + # The tactics are tuned on EPLB's three slots. + owner = weight_owner() + tuner_layout = {"tuner_num_local_experts": 3} + assert owner.tuner_layout_kwargs == tuner_layout + alpha_storage = weights["alpha"].data_ptr() + balancer.register_weight_slots_after_to_cuda() + balancer.finalize_model() + balancer.set_warm_up_iter_count(0) + assert { + name: id(tensor) + for name, tensor in layer.unbacked_slot_tensors.items() + } == { + "w": id(weights["w"]), + "s": id(weights["s"]) + } + if not layer_updates_per_iter: + assert weights["alpha"].data_ptr() == alpha_storage + + def route(experts, update=False): + """Route ``experts`` and return their slots, or update EPLB by them.""" + selected = torch.tensor(experts, dtype=torch.int32, + device="cuda")[:, None] + slots = None + with MoeLoadBalancerIterContext(balancer, update, update): + layer.start_wait_gpu_stage() + layer.done_wait_gpu_stage() + if update: + layer.update_statistic_with_global_ids(selected, True, True) + else: + slots = layer.route(selected).flatten().tolist() + layer.start_set_cpu_stage() + layer.done_set_cpu_stage() + return slots + + def assert_loaded(expert_ids): + for name in slot_elements: + for slot, expert_id in enumerate(expert_ids): + assert torch.equal(weights[name][slot].cpu(), + host[name][expert_id]), (name, slot) + + # Expert 3 is in rank 1's first slot. + assert set(route([3] * 8)) == {slot_count} + + if layer_updates_per_iter: + # A rank failure stopped the forward before its MoE layer, so the + # EPLB worker still waits for the layer. A restore writes no slot. + with MoeLoadBalancerIterContext(balancer, True, True): + pass + early = balancer.expert_restore.plan_expert_restore( + [True, False], [layer.get_old_rank_expert_ids()]) + spare = {name: weights[name][3].clone() for name in slot_elements} + with pytest.raises(RuntimeError, + match="abandonIterations|iterations are queued"): + balancer.expert_restore.restore_lost_experts(early) + assert all( + torch.equal(weights[name][3], spare[name]) + for name in slot_elements) + balancer.abandon_iterations() + plan = balancer.expert_restore.plan_expert_restore( + [True, False], [layer.get_old_rank_expert_ids()]) + # Backing spare slot 3 maps its four whole "w" chunks and the two "s" + # chunks holding its bytes, one shared with spare slot 4. Spare slots 4 + # and 5 stay unbacked. + assert plan.device_bytes == 8 * mib + 4 * mib + free = torch.cuda.mem_get_info()[0] + assert balancer.expert_restore.restore_lost_experts( + plan) == plan.device_bytes + assert free - torch.cuda.mem_get_info( + )[0] >= plan.device_bytes - scratch + # Only spare slot 3 is backed: backing it again maps nothing, backing slot 4 or 5 would. + assert spare_slot_bytes_to_back(layer, [3]) == 0 + assert all( + spare_slot_bytes_to_back(layer, [slot]) > 0 for slot in (4, 5)) + assert layer.get_old_rank_expert_ids()[0] == [0, 1, 2, 3, -1, -1] + assert_loaded(range(4)) + assert set(route([3] * 8)) == {3} + assert route([0, 1, 2]) == [0, 1, 2] + # A restore moves experts, never the tuner layout the tactics were tuned with. + assert owner.tuner_layout_kwargs == tuner_layout + assert weight_owner().tuner_layout_kwargs == tuner_layout + + if layer_updates_per_iter: + # Online EPLB balances over the restored spare slot as over the + # others. By load, experts 3, 0, 1 and 2 go to slots 0 to 3, so + # only the spare slot holds expert 2. + route([3] * 4 + [0] * 3 + [1] * 2, update=True) + balancer.abandon_iterations() + if layer_updates_per_iter: + assert layer.get_old_rank_expert_ids()[0] == [3, 0, 1, 2, -1, -1] + rejoin = balancer.expert_restore.plan_expert_restore( + [True, True], [layer.get_old_rank_expert_ids()], + rejoining=[False, True]) + assert rejoin.device_bytes == 0 + assert balancer.expert_restore.restore_lost_experts(rejoin) == 0 + free = torch.cuda.mem_get_info()[0] + assert balancer.expert_restore.release_empty_spare_slots( + ) == plan.device_bytes + assert torch.cuda.mem_get_info( + )[0] - free >= plan.device_bytes - scratch + # No spare slot is backed. + assert all( + spare_slot_bytes_to_back(layer, [slot]) > 0 + for slot in layer.spare_slots) + # The initial placement again: rank 0's active slots are loaded with their experts of it. + expected = [[0, 1, 2, -1, -1, -1], [3, 0, 1, -1, -1, -1]] + assert layer.get_old_rank_expert_ids() == expected + assert_loaded(range(3)) + # The expert in rank 1's first slot has no other replica. + assert set(route([expected[1][0]] * 8)) == {slot_count} + + if layer_updates_per_iter: + # The next iteration updates again, over the active slots of both + # ranks only. Expert 3 gets three replicas, then the experts go to + # the least loaded rank by load per replica: 2, 3, 3, 3, 1, 0. + route([3] * 10 + [2] * 4 + [1] * 2 + [0], update=True) + # The next forward waits for the update and routes by its placement. + assert route([2, 0, 1]) == [0, 2, slot_count + 2] + assert set(route([3] * 8)) <= {1, slot_count, slot_count + 1} + assert layer.get_old_rank_expert_ids() == [[2, 3, 0, -1, -1, -1], + [3, 3, 1, -1, -1, -1]] + assert_loaded([2, 3, 0]) + finally: + balancer.shutdown() + + +_EXPERTS, _HIDDEN, _INTERMEDIATE = 4, 2048, 2048 + + +def _vmm_granularity() -> int: + """The VMM granularity of the unbacked slot memory on the current device.""" + from cuda.bindings import driver + + prop = driver.CUmemAllocationProp() + prop.type = driver.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED + prop.location.type = driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + prop.location.id = torch.cuda.current_device() + error, granularity = driver.cuMemGetAllocationGranularity( + prop, driver.CUmemAllocationGranularity_flags. + CU_MEM_ALLOC_GRANULARITY_MINIMUM) + assert error == driver.CUresult.CUDA_SUCCESS + return granularity + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +def test_unbacked_slots_back_and_unback(): + """Back a spare slot in place and unback it, taking device memory only while it is backed.""" + from tensorrt_llm.bindings import MemoryCounters + + torch.cuda.set_device(0) + chunk = _vmm_granularity() + counters = MemoryCounters.instance() + before = counters.gpu + # Four slots of one VMM chunk each, of which slots 2 and 3 are spare. + slots = torch.ops.trtllm.moe_unbacked_slot_tensor([4, chunk], torch.uint8, + [2, 3]) + address = slots.data_ptr() + # The two backed slots and at most the scratch chunk the spare slots share. + assert counters.gpu - before <= 3 * chunk + assert torch.ops.trtllm.moe_unbacked_slots_bytes_to_back(slots, + [2]) == chunk + assert torch.ops.trtllm.moe_unbacked_slots_set_backed([slots], [2], [1], + True) == chunk + assert slots.data_ptr() == address + slots[2].fill_(7) + slots[3].fill_(1) + assert bool(slots[2].eq(7).all()) + backed = counters.gpu + assert torch.ops.trtllm.moe_unbacked_slots_set_backed([slots], [2], [1], + False) == chunk + assert backed - counters.gpu == chunk + del slots + assert counters.gpu == before + + +def _load_moe_layer(monkeypatch, fault_tolerance: bool, + meta_init: bool) -> dict: + """Load one Cutlass BF16 MoE layer as the model loader does and measure its memory. + + The memory counts the caching allocator's allocations and the unbacked slot + memory that VMM maps outside it. + """ + from tensorrt_llm._torch.models.modeling_utils import MetaInitMode + from tensorrt_llm._torch.moe.fused_moe import (RenormalizeMoeRoutingMethod, + create_moe) + from tensorrt_llm.bindings import MemoryCounters + + # Two ranks on two nodes. With fault tolerance, rank 0 alone must hold the + # four experts, two slots more than EPLB's two. + monkeypatch.setattr(EplbExpertRestore, "_rank_nodes", lambda self: [0, 1]) + eplb_config = MoeLoadBalancerConfig(num_slots=_EXPERTS, + layer_updates_per_iter=0) + eplb_config.setup(ep_rank=0, ep_size=2) + pretrained_config = PretrainedConfig() + pretrained_config.num_experts = _EXPERTS + pretrained_config.hidden_size = _HIDDEN + pretrained_config.intermediate_size = _INTERMEDIATE + pretrained_config.torch_dtype = torch.bfloat16 + model_config = ModelConfig(pretrained_config=pretrained_config, + mapping=Mapping(world_size=2, + rank=0, + tp_size=2, + moe_ep_size=2, + enable_attention_dp=True), + moe_backend="CUTLASS", + moe_load_balancer=eplb_config) + shapes = { + "w1": (_INTERMEDIATE, _HIDDEN), + "w3": (_INTERMEDIATE, _HIDDEN), + "w2": (_HIDDEN, _INTERMEDIATE) + } + weights = { + f"{expert_id}.{name}.weight": + torch.full(shape, expert_id + 1.0, dtype=torch.bfloat16) + for expert_id in range(_EXPERTS) + for name, shape in shapes.items() + } + counters = MemoryCounters.instance() + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + allocated = torch.cuda.memory_allocated() + unbacked_before = counters.gpu + balancer_cls = MoeLoadBalancer + if fault_tolerance: + balancer_cls = functools.partial(FaultTolerantMoeLoadBalancer, + mapping=None) + balancer = balancer_cls( + 0, 2, 0, shared_memory_base_name=f"test_load_memory_{os.getpid()}") + try: + with balancer, MetaInitMode() if meta_init else nullcontext(): + moe = create_moe( + routing_method=RenormalizeMoeRoutingMethod(top_k=2), + model_config=model_config, + layer_idx=0, + communication_method="ALLGATHER") + # The model loader puts the model on the device before it loads the weights. + moe._apply(lambda t: torch.empty_like(t, device="cuda") + if t.device.type == "meta" else t) + moe.to("cuda") + moe.load_weights([weights]) + moe.post_load_weights() + balancer.register_weight_slots_after_to_cuda() + balancer.finalize_model() + torch.cuda.synchronize() + unbacked = counters.gpu - unbacked_before + peak = torch.cuda.max_memory_allocated() - allocated + unbacked + (layer, ) = balancer.single_layer_load_balancers + return { + "peak": peak, + "after": torch.cuda.memory_allocated() - allocated + unbacked, + "unbacked": sorted(layer.unbacked_slot_tensors), + "loaded": list(moe.backend.initial_local_expert_ids), + "w2": moe.backend.w2_weight[:2].float().mean(dim=(1, 2)).tolist(), + } + finally: + balancer.shutdown() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +@pytest.mark.parametrize("meta_init", [True, False], + ids=["meta_init", "regular_init"]) +def test_spare_slots_unbacked_while_loading(monkeypatch, meta_init): + """Load a layer in the device memory it takes without fault tolerance.""" + torch.cuda.set_device(0) + # The first load also makes what a process allocates once. + _load_moe_layer(monkeypatch, fault_tolerance=False, meta_init=meta_init) + gc.collect() + with_spares = _load_moe_layer(monkeypatch, + fault_tolerance=True, + meta_init=meta_init) + gc.collect() + without = _load_moe_layer(monkeypatch, + fault_tolerance=False, + meta_init=meta_init) + # Rank 0 loads its two experts; its two spare slots stay empty. + assert with_spares["loaded"] == without["loaded"] == [0, 1] + assert with_spares["w2"] == without["w2"] == [1.0, 2.0] + assert with_spares["unbacked"] == ["w2_weight", "w3_w1_weight"] + assert without["unbacked"] == [] + tolerance = (len(with_spares["unbacked"]) + 1) * _vmm_granularity() + # Two spare slots of w3_w1 and w2 in bfloat16, which loading took before. + assert 2 * 3 * _HIDDEN * _INTERMEDIATE * 2 > 4 * tolerance + for key in ("peak", "after"): + assert abs(with_spares[key] - + without[key]) <= tolerance, (key, with_spares[key], + without[key]) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +@pytest.mark.parametrize("fault_tolerance, slot_memory", [ + (False, "device"), + (True, "device"), + (True, "backed"), + (True, "unbacked_device"), + (True, "unbacked"), +]) +def test_cpu_copied_slot_outside_host_accessible_memory(fault_tolerance, + slot_memory): + """Reject a CPU-copied slot outside host accessible memory at finalize.""" + from tensorrt_llm.bindings.internal import runtime + + if slot_memory == "unbacked": + _skip_without_gdrcopy_slots() + torch.cuda.set_device(0) + host = torch.ones(16, dtype=torch.uint8) + + def weight(t: torch.Tensor) -> runtime.MoeWeight: + moe_weight = runtime.MoeWeight() + moe_weight.weight_ptr = t.data_ptr() + moe_weight.height, moe_weight.width, moe_weight.pitch = 1, 16, 16 + return moe_weight + + # The load balancer initializes host accessible device memory first. + balancer = runtime.MoeLoadBalancer(0, 1, 1, fault_tolerance=fault_tolerance) + storage = None + try: + if slot_memory == "device": + slot = torch.zeros(16, dtype=torch.uint8, device="cuda") + else: + # Slots larger than the mapping granularity, so slot 1 maps no + # memory. + storage = torch.ops.trtllm.moe_unbacked_slot_tensor( + [2, 4 << 20], torch.uint8, [1], slot_memory == "unbacked") + slot = storage[0 if slot_memory == "backed" else 1] + layer = balancer.add_layer(1, 1, 1) + layer.add_single_host_weight(0, "w", weight(host)) + layer.add_single_weight_slot(0, "w", weight(slot)) + layer.set_initial_weight_assignments([0]) + if slot_memory == "unbacked": + balancer.finalize_model() + else: + with pytest.raises(RuntimeError, + match="slotId=0, name=w not added"): + balancer.finalize_model() + finally: + # Release the slots while host accessible device memory is set up. + slot = storage = None + balancer.shutdown() + + if __name__ == '__main__': unittest.main() diff --git a/tests/unittest/_torch/moe/test_spare_slots.py b/tests/unittest/_torch/moe/test_spare_slots.py new file mode 100644 index 000000000000..34a0952c0178 --- /dev/null +++ b/tests/unittest/_torch/moe/test_spare_slots.py @@ -0,0 +1,412 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the spare EPLB slots that hold a failed rank's experts on the active ranks.""" + +import functools +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import numpy as np +import pytest +import torch + +from tensorrt_llm._torch.moe.fused_moe.cutedsl import TrtllmCutedslGroupedGemmNvfp4Impl +from tensorrt_llm._torch.moe.fused_moe.cutlass import ( + DeepgemmCudaHopperFp8BlockScalesImpl, + TrtllmCutlassNvfp4Impl, + TrtllmCutlassUnquantizedImpl, + TrtllmCutlassW4a16Nvfp4Impl, +) +from tensorrt_llm._torch.moe.fused_moe.eplb_expert_restore import ( + EplbExpertRestore, + FaultTolerantMoeLoadBalancer, + create_spare_slot_storage, + plan_layer_placement, + use_spare_slot_storage, +) +from tensorrt_llm._torch.moe.fused_moe.impl_blocks import ( + MoEEplbWeightLayoutMixin, + MoEWeightOwnerMixin, +) +from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import MoeLoadBalancer + +pytestmark = pytest.mark.cpu_only + +_ER = "tensorrt_llm._torch.moe.fused_moe.eplb_expert_restore" +_USES_GDRCOPY = "tensorrt_llm.bindings.internal.runtime.host_accessible_device_memory_uses_gdrcopy" +WORLD_SIZE = 4 +EXPERTS = 4 + + +def _deployment(): + """Return the keyword arguments of a rank's load balancer.""" + return {"mapping": None} + + +def _fault_tolerance(monkeypatch, largest_failure): + """Turn fault tolerance on for nodes [0, 1] and [2, 3] and ``largest_failure`` lost ranks.""" + monkeypatch.setattr(EplbExpertRestore, "_rank_nodes", lambda self: [0, 0, 1, 1]) + recoverable = EplbExpertRestore._recoverable_failures + monkeypatch.setattr( + EplbExpertRestore, + "_recoverable_failures", + staticmethod(lambda nodes: [f for f in recoverable(nodes) if len(f) <= largest_failure]), + ) + + +def _load_balancer( + monkeypatch, *, largest_failure, eplb_slots, layer_updates_per_iter=1, ep_rank=0 +): + """Return a fault-tolerant MoeLoadBalancer over one layer of ``eplb_slots`` per rank.""" + _fault_tolerance(monkeypatch, largest_failure) + with ( + patch("tensorrt_llm.bindings.internal.runtime.MoeLoadBalancer"), + patch(_USES_GDRCOPY, return_value=True), + patch(f"{_ER}.DeploymentHostWeights"), + patch("torch.cuda.Event"), + ): + load_balancer = FaultTolerantMoeLoadBalancer( + ep_rank, WORLD_SIZE, layer_updates_per_iter, **_deployment() + ) + load_balancer.add_layer(EXPERTS, 2, eplb_slots, aux_stream=Mock()) + return load_balancer + + +@pytest.mark.parametrize( + "largest_failure, layer_updates_per_iter, uses_gdrcopy, refused", + [ + (2, 1, False, True), + (2, 1, True, False), + # Static EPLB writes no slot from the host. Without a rank to lose there is no spare slot. + (2, 0, False, False), + (0, 1, False, False), + ], +) +def test_online_spare_slots_need_gdrcopy( + monkeypatch, largest_failure, layer_updates_per_iter, uses_gdrcopy, refused +): + """Refuse online EPLB with spare slots unless GDRCopy maps the slots for host writes.""" + _fault_tolerance(monkeypatch, largest_failure) + with ( + patch("tensorrt_llm.bindings.internal.runtime.MoeLoadBalancer"), + patch(_USES_GDRCOPY, return_value=uses_gdrcopy), + pytest.raises(ValueError, match="requires GDRCopy") if refused else nullcontext(), + ): + FaultTolerantMoeLoadBalancer(0, WORLD_SIZE, layer_updates_per_iter, **_deployment()) + + +@pytest.mark.parametrize( + "largest_failure, eplb_slots, spare_slot_count", + [ + # After losing a node of two, the two active ranks need two slots each for four experts. + (2, 1, 1), + # After losing one rank, the three active ranks still need two slots each. + (1, 1, 1), + # A node may hold both replicas of up to four experts, which the spare slots take. + (2, 2, 2), + # Without a rank to lose, no spare slot is added. + (0, 1, 0), + ], +) +def test_spare_slots_cover_largest_failure( + monkeypatch, largest_failure, eplb_slots, spare_slot_count +): + """Add enough spare slots for the survivors of the largest failure to hold every expert.""" + load_balancer = _load_balancer( + monkeypatch, largest_failure=largest_failure, eplb_slots=eplb_slots + ) + (layer,) = load_balancer.single_layer_load_balancers + load_balancer.load_balancer_impl.add_layer.assert_called_once_with( + EXPERTS, 2, eplb_slots + spare_slot_count + ) + assert layer.spare_slot_count == spare_slot_count + assert layer.active_slot_count_per_rank == eplb_slots + assert load_balancer.adds_spare_slots is (spare_slot_count > 0) + + +def test_no_spare_slots_without_fault_tolerance(monkeypatch): + monkeypatch.setattr(EplbExpertRestore, "_rank_nodes", Mock(side_effect=AssertionError("nodes"))) + with ( + patch("tensorrt_llm.bindings.internal.runtime.MoeLoadBalancer"), + patch.object(MoeLoadBalancer, "host_tensor_sharer_cls"), + patch("torch.cuda.Event"), + ): + load_balancer = MoeLoadBalancer(0, WORLD_SIZE, 1) + layer = load_balancer.add_layer(EXPERTS, 2, 1, aux_stream=Mock()) + load_balancer.load_balancer_impl.add_layer.assert_called_once_with(EXPERTS, 2, 1) + assert layer.spare_slot_count == 0 + assert not load_balancer.adds_spare_slots + assignments = [0, 1, 2, 3] + assert layer.with_spare_slots(assignments) is assignments + layer.set_initial_weight_assignments(assignments) + assert layer.initial_placement is None + + +@pytest.mark.parametrize("misuse", ["storage_replaced", "no_host_weights"]) +def test_spare_storage_misuse_refused(monkeypatch, misuse): + """Refuse a weight whose spare slots cannot be restored, for lost storage or no host weights.""" + load_balancer = _load_balancer( + monkeypatch, largest_failure=2, eplb_slots=1, layer_updates_per_iter=0 + ) + (layer,) = load_balancer.single_layer_load_balancers + with patch(f"{_ER}.torch") as mock_torch: + mock_torch.ops.trtllm.moe_unbacked_slots_save_memory.return_value = True + mock_torch.ops.trtllm.moe_unbacked_slot_tensor.return_value = torch.zeros(2, 4) + create_spare_slot_storage(layer, "w", torch.zeros(2, 4)) + if misuse == "no_host_weights": + message = r"Weights \['w'\] of MoE layer .* no host weights" + else: + layer.add_to_migrate_weight_fn(use_spare_slot_storage, (layer, torch.zeros(2, 4), "w")) + message = "Weight w of MoE layer .* lost the storage" + with pytest.raises(RuntimeError, match=message): + load_balancer.register_weight_slots_after_to_cuda() + + +@pytest.mark.parametrize("fault_tolerance", [True, False], ids=["fault_tolerance", "off"]) +def test_fault_tolerant_shutdown_skips_node_collectives(monkeypatch, fault_tolerance): + """Skip the node's allgather and barrier at shutdown under fault tolerance only.""" + load_balancer = _load_balancer(monkeypatch, largest_failure=2, eplb_slots=1) + (layer,) = load_balancer.single_layer_load_balancers + # Without fault tolerance, the load balancer's shutdown is MoeLoadBalancer's. + shutdown = ( + load_balancer.shutdown + if fault_tolerance + else functools.partial(MoeLoadBalancer.shutdown, load_balancer) + ) + node = Mock(allgather=Mock(side_effect=lambda safe: [safe, True])) + load_balancer.shared_mpi_comm = node + events = [] + load_balancer.load_balancer_impl.shutdown.side_effect = lambda: events.append("shutdown") + layer.host_tensor_sharer.pre_shutdown_cleanup.side_effect = lambda: events.append("release") + load_balancer.in_iter = True + assert not shutdown() + assert events == [] + load_balancer.in_iter = False + assert shutdown() + assert events == ["shutdown", "release"] + assert node.allgather.call_count == (0 if fault_tolerance else 2) + assert node.barrier.call_count == (0 if fault_tolerance else 2) + + +def _placement(*rows): + return np.array(rows, dtype=np.int32) + + +def test_failed_rank_experts_move_and_return(): + """Plan the placement as ranks 3 and 2 fail and then rejoin.""" + initial = _placement([0, 1, -1, -1], [2, 3, -1, -1], [4, 5, -1, -1], [6, 7, -1, -1]) + + def plan(current, active, layer=0): + return plan_layer_placement(current, initial, np.array(active, bool), 2, 8, layer) + + # Rank 3 fails, and its experts go to the spare slots of the ranks with the fewest filled. + without_3 = plan(initial, [1, 1, 1, 0]) + np.testing.assert_array_equal( + without_3, _placement([0, 1, 6, -1], [2, 3, 7, -1], [4, 5, -1, -1], [-1] * 4) + ) + # The next layer starts at the next rank, so that the layers spread the lost experts. + np.testing.assert_array_equal( + plan(initial, [1, 1, 1, 0], layer=1), + _placement([0, 1, -1, -1], [2, 3, 6, -1], [4, 5, 7, -1], [-1] * 4), + ) + # Rank 2 fails too, and the experts restored before stay where they are. + without_2_3 = plan(without_3, [1, 1, 0, 0]) + np.testing.assert_array_equal( + without_2_3, _placement([0, 1, 6, 4], [2, 3, 7, 5], [-1] * 4, [-1] * 4) + ) + # When rank 2 rejoins, the spare slots holding its experts are emptied. + np.testing.assert_array_equal(plan(without_2_3, [1, 1, 1, 0]), without_3) + # With every rank back, the placement is the initial one. + np.testing.assert_array_equal(plan(without_3, [1, 1, 1, 1]), initial) + # Without spare slots, lost experts take the active slots of extra replicas. + replicated = _placement([0, 1], [2, 3], [0, 1]) + np.testing.assert_array_equal( + plan_layer_placement(replicated, replicated, np.array([1, 0, 1], bool), 2, 4, 0), + _placement([0, 1], [-1, -1], [2, 3]), + ) + # Without spare slots or extra replicas, a lost expert has no slot. + with pytest.raises(ValueError, match=r"active ranks \[0\] cannot hold expert 2"): + plan_layer_placement(initial[:2, :2], initial[:2, :2], np.array([1, 0], bool), 2, 4, 0) + + +def test_single_replica_layer_keeps_one_replica_per_expert(): + """Plan a single replica per expert from ranks that an online EPLB update left mixed.""" + # A swap of experts 0 and 2 reached rank 0 only, so rank 1's slot of expert 2 takes expert 0. + mixed = _placement([2, 1, -1, -1], [2, 3, -1, -1], [4, 5, -1, -1], [6, 7, -1, -1]) + placement = plan_layer_placement(mixed, mixed, np.array([1, 1, 1, 0], bool), 2, 8, 0) + np.testing.assert_array_equal( + placement, _placement([2, 1, 6, -1], [0, 3, 7, -1], [4, 5, -1, -1], [-1] * 4) + ) + + +def test_multi_replica_layer_restores_experts_into_extra_replicas(): + """Plan every expert from ranks that an online EPLB update left mixed in a multi-replica layer.""" + # An update moving expert 2 to rank 1 reached rank 0 only. Four experts lack a replica for + # three spare slots, so expert 8 takes rank 1's slot of an extra replica of expert 4. + mixed = _placement([0, 1, 4, -1], [3, 4, 5, -1], [0, 3, 1, -1], [6, 7, 8, -1]) + placement = plan_layer_placement(mixed, mixed, np.array([1, 1, 1, 0], bool), 3, 9, 0) + np.testing.assert_array_equal( + placement, _placement([0, 1, 4, 2], [3, 8, 5, 6], [0, 3, 1, 7], [-1] * 4) + ) + + +def _online_balancer(monkeypatch, current, ep_rank=0): + """Return an online EPLB load balancer at placement ``current`` with 4 MiB spare slots.""" + load_balancer = _load_balancer(monkeypatch, largest_failure=2, eplb_slots=2, ep_rank=ep_rank) + (layer,) = load_balancer.single_layer_load_balancers + layer.set_initial_weight_assignments(layer.with_spare_slots([0, 1, 2, 3, 0, 1, 2, 3])) + layer.single_layer_load_balancer_impl.get_old_rank_expert_ids.return_value = current.tolist() + layer.unbacked_slot_tensors = {"w": Mock()} + monkeypatch.setattr( + torch.ops.trtllm, + "moe_unbacked_slots_bytes_to_back", + lambda tensor, slots: len(slots) * (4 << 20), + raising=False, + ) + return load_balancer, layer + + +# Online EPLB moved the experts away from the initial placement. +_ONLINE = _placement([1, 0, -1, -1], [3, 2, -1, -1], [2, 0, -1, -1], [1, 2, -1, -1]) +_ONLINE_WITHOUT_1 = _placement([1, 0, 3, -1], [-1] * 4, [2, 0, -1, -1], [1, 2, -1, -1]) + + +def test_active_ranks_plan_the_same_restore(monkeypatch): + """Plan the same restore on active ranks with different views of the placement.""" + active = [True, False, True, True] + # An interrupted online update left rank 2 with another view of the placement than rank 0's. + views = { + 0: _ONLINE, + 2: _placement([1, 0, -1, -1], [3, 2, -1, -1], [2, 0, -1, -1], [3, 2, -1, -1]), + } + plans = [ + _online_balancer(monkeypatch, view, ep_rank)[0].expert_restore.plan_expert_restore( + active, [_ONLINE] + ) + for ep_rank, view in views.items() + ] + for plan in plans: + # The active ranks' active slots keep their experts, and expert 3 goes to a spare slot. + np.testing.assert_array_equal(plan.placements[0], _ONLINE_WITHOUT_1) + assert plans[0].digest == plans[1].digest + # Rank 0 backs one 4 MiB slot and rank 2 backs none. + assert [plan.device_bytes for plan in plans] == [4 << 20, 0] + other = _online_balancer(monkeypatch, _ONLINE)[0].expert_restore.plan_expert_restore( + [True, True, False, True], [_ONLINE] + ) + assert other.digest != plans[0].digest + + +def test_release_keeps_filled_spare_slots_during_update(monkeypatch): + """Release only the empty spare slots while an online EPLB update swaps the placement.""" + # The old placement reads -1 during the swap, so only the unusable slots tell 2 from 3. + load_balancer, layer = _online_balancer(monkeypatch, _placement(*[[-1] * 4] * WORLD_SIZE)) + layer.single_layer_load_balancer_impl.get_unusable_local_slots.return_value = [3] + released = [] + + def set_backed(tensors, slots, slot_counts, backed): + assert not backed + released.extend(slots) + return len(slots) * (4 << 20) + + monkeypatch.setattr( + torch.ops.trtllm, "moe_unbacked_slots_set_backed", set_backed, raising=False + ) + assert load_balancer.expert_restore.release_empty_spare_slots() == 4 << 20 + assert released == [3] + + +def test_restore_reads_no_placement_while_an_update_may_swap_it(monkeypatch): + """Refuse even a restore that changes nothing before reading the placement an update swaps.""" + load_balancer, layer = _online_balancer(monkeypatch, _ONLINE) + restore = load_balancer.expert_restore + # Every rank is active and keeps its experts, so the plan loads no slot. + plan = restore.plan_expert_restore([True] * WORLD_SIZE, [_ONLINE]) + np.testing.assert_array_equal(plan.placements[0], _ONLINE) + load_balancer.load_balancer_impl.check_slots_writable.side_effect = RuntimeError( + "call abandonIterations first" + ) + old_rank_expert_ids = layer.single_layer_load_balancer_impl.get_old_rank_expert_ids + old_rank_expert_ids.reset_mock() + with pytest.raises(RuntimeError, match="abandonIterations"): + restore.restore_lost_experts(plan) + old_rank_expert_ids.assert_not_called() + + +class _SpareSlotOwner(MoEWeightOwnerMixin, MoEEplbWeightLayoutMixin): + """A MoE layer's weight owner with the quantization method of the MoE backend ``leaf``.""" + + def __init__(self, leaf, spare_slot_count): + self.layer_load_balancer = SimpleNamespace(spare_slot_count=spare_slot_count) + self.method = leaf.__new__(leaf)._get_quant_method() + self.method.create_weights = Mock() + + def _get_quant_method(self): + return self.method + + +@pytest.mark.parametrize("spare_slot_count", [2, 0]) +def test_spare_slot_weights_created_after_the_expert_weights(monkeypatch, spare_slot_count): + """Create the spare slots' weights after the quantization method's, only with spare slots.""" + create_spare_slot_weights = Mock() + monkeypatch.setattr(f"{_ER}.create_spare_slot_weights", create_spare_slot_weights) + owner = _SpareSlotOwner(TrtllmCutlassNvfp4Impl, spare_slot_count) + owner.create_weights() + owner.method.create_weights.assert_called_once_with(owner) + assert create_spare_slot_weights.call_count == (1 if spare_slot_count else 0) + + +@pytest.mark.parametrize( + "leaf, quant, sm, rejected", + [ + (TrtllmCutlassNvfp4Impl, "NVFP4", 100, False), + (TrtllmCutlassUnquantizedImpl, None, 100, False), + (TrtllmCutedslGroupedGemmNvfp4Impl, "NVFP4", 100, False), + # The SM107 (Rubin) kernels tune every local slot. + (TrtllmCutedslGroupedGemmNvfp4Impl, "NVFP4", 107, True), + (TrtllmCutlassW4a16Nvfp4Impl, "W4A16_NVFP4", 100, True), + (DeepgemmCudaHopperFp8BlockScalesImpl, "FP8_BLOCK_SCALES", 90, True), + ], +) +def test_spare_slots_checked_in_can_implement(leaf, quant, sm, rejected): + """A backend that would autotune or allocate for the spare slots declines them.""" + import dataclasses + + from tensorrt_llm._torch.moe.fused_moe.impl_contract import ( + MoEDeployment, + MoEEnvironment, + MoEProblem, + MoERejectReason, + ) + from tensorrt_llm._torch.moe.fused_moe.impl_environment import MoEDep + + problem = MoEProblem(quant=quant, dtype_act=torch.bfloat16) + deployment = MoEDeployment( + ep_size=4, + tp_size=1, + parallel_size=4, + use_dp=True, + num_slots=8, + env=MoEEnvironment(sm=sm, available_deps=(MoEDep.CUTEDSL_RUBIN.value,)), + eplb_enabled=True, + eplb_mode="standard", + ) + assert leaf.can_implement(problem, deployment).eligible + + verdict = leaf.can_implement(problem, dataclasses.replace(deployment, spare_slots_enabled=True)) + assert verdict.eligible is not rejected + if rejected: + assert verdict.reject_reason is MoERejectReason.SPARE_SLOTS_UNSUPPORTED diff --git a/tests/unittest/_torch/moe/test_w4a16_nvfp4_eplb.py b/tests/unittest/_torch/moe/test_w4a16_nvfp4_eplb.py index 4a5a046b0327..9830c943c82c 100644 --- a/tests/unittest/_torch/moe/test_w4a16_nvfp4_eplb.py +++ b/tests/unittest/_torch/moe/test_w4a16_nvfp4_eplb.py @@ -21,7 +21,9 @@ def test_w4a16_eplb_migrates_linear_block_scales(gated: bool, streamed: bool) -> module.tp_size = 1 module.tp_rank = 0 module.initial_local_expert_ids = [2, 0] - module.layer_load_balancer = SimpleNamespace(get_load_expert_ids=lambda: [0, 1, 2, 3]) + module.layer_load_balancer = SimpleNamespace( + get_load_expert_ids=lambda: [0, 1, 2, 3], spare_slot_count=0 + ) fc1_rows = 256 if gated else 128 module.w3_w1_weight_scale = torch.nn.Parameter( torch.zeros(2, fc1_rows, 4, dtype=torch.int32), requires_grad=False From 1650f13c908f891d82e7bc450d6e6d06614f6b8a Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Thu, 19 Nov 2026 14:52:41 -0800 Subject: [PATCH 06/13] [None][feat] Wide-EP fault tolerance: membership record, authority process and rendezvous store PR 3 of 7 of Wide-EP fault tolerance. Every surviving rank must agree on which ranks are active, at which generation, and where to resume, and a failed or hung process must never change that by itself. - Adds the membership authority: one CPU-only process on node 0 that decides the membership. Ranks report progress and waits to it; it detects crashes and hangs, drives each membership change and publishes the result. - Crash detection is a closed TCP liveness connection, so a crash is seen in milliseconds. - Hang rule: only the end of a wait-for chain that made no progress for the hang timeout (60 s) is removed, never the ranks that wait on it. A rank that its node supervisor reports paused is not hung. - Adds a signed rendezvous store and StoreGroup, an mpi4py-style communicator over the store whose exchanges reach only the active members. - New package only; the single change to existing code is a refactor of the DWDP store rendezvous. Signed-off-by: Andrew Rusli --- .../_torch/distributed/membership/__init__.py | 106 + .../_torch/distributed/membership/__main__.py | 21 + .../distributed/membership/authority.py | 636 ++++++ .../membership/authority_process.py | 654 ++++++ .../_torch/distributed/membership/liveness.py | 417 ++++ .../_torch/distributed/membership/records.py | 284 +++ .../distributed/membership/rendezvous.py | 265 +++ .../distributed/membership/signed_store.py | 174 ++ .../_torch/modules/dwdp/rendezvous.py | 80 +- tensorrt_llm/_wide_ep_fault_tolerance.py | 4 + .../distributed/store_group_test_utils.py | 75 + .../_torch/distributed/test_membership.py | 1756 +++++++++++++++++ .../distributed/test_membership_hang_rule.py | 383 ++++ .../_torch/distributed/test_store_group.py | 259 +++ 14 files changed, 5085 insertions(+), 29 deletions(-) create mode 100644 tensorrt_llm/_torch/distributed/membership/__init__.py create mode 100644 tensorrt_llm/_torch/distributed/membership/__main__.py create mode 100644 tensorrt_llm/_torch/distributed/membership/authority.py create mode 100644 tensorrt_llm/_torch/distributed/membership/authority_process.py create mode 100644 tensorrt_llm/_torch/distributed/membership/liveness.py create mode 100644 tensorrt_llm/_torch/distributed/membership/records.py create mode 100644 tensorrt_llm/_torch/distributed/membership/rendezvous.py create mode 100644 tensorrt_llm/_torch/distributed/membership/signed_store.py create mode 100644 tests/unittest/_torch/distributed/store_group_test_utils.py create mode 100644 tests/unittest/_torch/distributed/test_membership.py create mode 100644 tests/unittest/_torch/distributed/test_membership_hang_rule.py create mode 100644 tests/unittest/_torch/distributed/test_store_group.py diff --git a/tensorrt_llm/_torch/distributed/membership/__init__.py b/tensorrt_llm/_torch/distributed/membership/__init__.py new file mode 100644 index 000000000000..e1a9e8c9f2a0 --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/__init__.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Membership for Wide-EP fault tolerance. + +The authority process decides which ranks are active and publishes each decision as a membership +record. Every rank keeps a liveness connection to it, and the ranks exchange values through the +rendezvous store it hosts. +""" + +from tensorrt_llm._wide_ep_fault_tolerance import RENDEZVOUS_ENV, SHARED_SECRET_ENV + +from .authority import Authority +from .authority_process import AuthorityProcess +from .liveness import ( + MembershipClient, + open_liveness_connection, + open_node_supervisor_connection, + open_store, + rendezvous_store, +) +from .records import ( + AUTHORITY_SILENCE_TIMEOUT_S, + DEAD_TIMEOUT_S, + HANG_TIMEOUT_S, + HELLO_TIMEOUT_S, + RENDEZVOUS_TIMEOUT_S, + REPORT_INTERVAL_S, + MembershipRecord, + MembershipState, + NodeReport, + NodeView, + RankReport, + Reply, + ResumeState, + TransactionSummary, + all_ranks, + has_rank, + latest_epoch, + next_epoch, + rank_mask, + remove_cpu_affinity_constraints, +) +from .rendezvous import ( + LIVENESS_KEY, + RECORD_KEY, + STALLED_SINCE_KEY, + TRANSACTION_KEY, + StoreGroup, + rendezvous_host_port, + wait_for_keys, +) +from .signed_store import SignedStore, UnsignedValueError, parse_shared_secret + +__all__ = [ + "AUTHORITY_SILENCE_TIMEOUT_S", + "Authority", + "AuthorityProcess", + "DEAD_TIMEOUT_S", + "HANG_TIMEOUT_S", + "HELLO_TIMEOUT_S", + "LIVENESS_KEY", + "MembershipClient", + "MembershipRecord", + "MembershipState", + "NodeReport", + "NodeView", + "RECORD_KEY", + "RENDEZVOUS_ENV", + "RENDEZVOUS_TIMEOUT_S", + "REPORT_INTERVAL_S", + "RankReport", + "Reply", + "ResumeState", + "SHARED_SECRET_ENV", + "STALLED_SINCE_KEY", + "SignedStore", + "StoreGroup", + "TRANSACTION_KEY", + "TransactionSummary", + "UnsignedValueError", + "all_ranks", + "has_rank", + "latest_epoch", + "next_epoch", + "open_liveness_connection", + "open_node_supervisor_connection", + "open_store", + "parse_shared_secret", + "rank_mask", + "remove_cpu_affinity_constraints", + "rendezvous_host_port", + "rendezvous_store", + "wait_for_keys", +] diff --git a/tensorrt_llm/_torch/distributed/membership/__main__.py b/tensorrt_llm/_torch/distributed/membership/__main__.py new file mode 100644 index 000000000000..0304c6784c21 --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/__main__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run the Wide-EP fault-tolerance membership authority.""" + +import sys + +from .authority_process import main + +sys.exit(main()) diff --git a/tensorrt_llm/_torch/distributed/membership/authority.py b/tensorrt_llm/_torch/distributed/membership/authority.py new file mode 100644 index 000000000000..4b422eae40dd --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/authority.py @@ -0,0 +1,636 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Membership authority for Wide-EP fault tolerance. + +:class:`Authority` decides membership from the ranks' reports. It does no I/O and takes monotonic +time from its callers, so tests drive it with simulated ranks. It reads the wall clock only to +report :class:`TransactionSummary` times in Unix milliseconds. +""" + +import time +from collections.abc import Iterable +from typing import Optional + +from tensorrt_llm.logger import logger + +from .records import ( + ADMISSION_TIMEOUT_S, + HANG_TIMEOUT_S, + MAX_RANKS, + RENDEZVOUS_TIMEOUT_S, + REPORT_INTERVAL_S, + MembershipRecord, + MembershipState, + NodeReport, + RankReport, + Reply, + ResumeState, + TransactionSummary, + all_ranks, + has_rank, + latest_epoch, + next_epoch, + rank_mask, +) + + +def _ranks(mask: int) -> list[int]: + """Return the ranks set in ``mask``, lowest first.""" + return [r for r in range(mask.bit_length()) if has_rank(mask, r)] + + +def _chain_ends(ranks: Iterable[int], waits: dict[int, int]) -> int: + """Return a mask of the ``ranks`` that are the end of a wait-for chain, so they may be hung. + + ``waits`` maps each rank to a mask of the ranks it waits on. A rank is an end when every rank + it reaches through ``waits`` also reaches it, so it waits on nobody or only within its cycle. + A cycle forms when a stopped process's last report names a peer that now waits on it. In + ``0 -> 2 -> 3 -> 2``, ranks 2 and 3 are ends and rank 0 is not. A rank that waits on a rank + that waits on nobody is never an end. + + The ends are the strongly connected components that no wait leaves. Tarjan's algorithm finds + the components in one depth-first search over the waits. + """ + order: dict[int, int] = {} + low: dict[int, int] = {} + stack: list[int] = [] + # The ranks whose component is found, and the ends among them. + done = ends = 0 + for root in ranks: + if root in order: + continue + order[root] = low[root] = len(order) + stack.append(root) + path = [(root, iter(_ranks(waits.get(root, 0))))] + while path: + r, peers = path[-1] + q = next(peers, None) + if q is not None: + if q not in order: + order[q] = low[q] = len(order) + stack.append(q) + path.append((q, iter(_ranks(waits.get(q, 0))))) + elif not has_rank(done, q): + low[r] = min(low[r], order[q]) + continue + path.pop() + if path: + parent = path[-1][0] + low[parent] = min(low[parent], low[r]) + if low[r] != order[r]: + continue + component = 0 + while not has_rank(component, r): + component |= 1 << stack.pop() + done |= component + if not any(waits.get(q, 0) & ~component for q in _ranks(component)): + ends |= component + return ends + + +def _update_last_change(seen: dict[int, tuple], rank: int, value: tuple, now_s: float) -> float: + """Store ``value`` for ``rank`` in ``seen`` if it changed and return when it last changed.""" + if seen.get(rank, (None,))[0] != value: + seen[rank] = (value, now_s) + return seen[rank][1] + + +class Authority: + """Decide membership from the ranks' reports. + + A rank fails when its connection closes, or when it is the end of a wait-for chain that makes no + progress for the timeout. Any failure while STARTING aborts. A replacement that fails to join is + left out. + + Outside a change, a rank whose node supervisor reports its hang detector paused, in a report + received within half the hang timeout, is never removed as hung, also once the rank's own + reports stop. The ranks that wait on such a rank stall until their own hang detector or + all-to-all timeout ends them. + + Call :meth:`tick` every report interval. ``record`` holds the current decision and + ``last_transaction`` summarizes the last ended change. The authority process publishes + ``stalled_since_s`` and ``stalled_replacements`` for health reporting. + """ + + def __init__( + self, + world_size: int, + hang_timeout_s: float = HANG_TIMEOUT_S, + rendezvous_timeout_s: float = RENDEZVOUS_TIMEOUT_S, + admission_timeout_s: float = ADMISSION_TIMEOUT_S, + ) -> None: + """Start with every rank of the deployment in a STARTING record. + + Args: + world_size: The deployment's ranks, at most ``MAX_RANKS``. + hang_timeout_s: How long a rank that others wait on may make no progress before it is + removed as hung. + rendezvous_timeout_s: How long the startup waits for every rank to connect, and how + long a starting rank or a replacement's startup may make no progress. + admission_timeout_s: How long a joining rank that waits on no peer may make no progress + in its admission before it is left out. + + Raises: + ValueError: ``world_size`` is not in ``(0, MAX_RANKS]``. + """ + if not 0 < world_size <= MAX_RANKS: + raise ValueError(f"world_size must be in (0, {MAX_RANKS}], got {world_size}") + self.world_size = world_size + self.hang_timeout_s = hang_timeout_s + self.rendezvous_timeout_s = rendezvous_timeout_s + self.admission_timeout_s = admission_timeout_s + self.record = MembershipRecord( + generation=0, state=MembershipState.STARTING, active_mask=all_ranks(world_size) + ) + self._committed_mask = self.record.active_mask + self._rendezvous_began_s: Optional[float] = None + # The ranks that connected at least once during the startup. + self._connected_once = 0 + # rank -> (value, time it last changed), for the progress and the startup progress. + self._progress: dict[int, tuple] = {} + self._startup_progress: dict[int, tuple] = {} + self._hung_since: dict[int, float] = {} + # rank -> when the latest node report that counted it as paused arrived. + self._last_paused_s: dict[int, float] = {} + # rank -> the first decision time at which it had no report. + self._unreported_since: dict[int, float] = {} + # The ranks whose replacement asked to join and is not in a proposed change yet. + self._requested: set[int] = set() + # The replacements in the current change, until it commits or leaves them out. + self._joining: set[int] = set() + # rank -> generation of the change that removed it, so replacements for ranks that failed + # together join in one change. + self._failed_in: dict[int, int] = {} + # Resume epoch of an INSTALL that a newer change replaced before it committed, or 0. + self._replaced_install_epoch = 0 + # When the pending change's first failure began, or its first replacement was ready. + self._detected_s: Optional[float] = None + self._causes: dict[int, str] = {} + # Since when the active ranks have waited on a rank that makes no progress, for longer than + # a report interval. + self.stalled_since_s: Optional[float] = None + self._transaction_began_s: Optional[float] = None + self._ready_s: Optional[float] = None + self.last_transaction: Optional[TransactionSummary] = None + self.stalled_replacements: tuple[int, ...] = () + self._changes: dict[str, int] = {} + + def request_join(self, rank: int) -> None: + """Request that a replacement of ``rank``, which the membership no longer lists, join. + + The replacement joins once the change that removes the old process has committed. + """ + self._requested.add(rank) + + def tick( + self, + now_s: float, + reports: dict[int, RankReport], + connected: int, + node_reports: Iterable[NodeReport] = (), + ) -> Optional[MembershipRecord]: + """Decide membership at monotonic ``now_s`` and return a new record to publish, or None. + + ``connected`` masks the ranks whose current process has an open liveness connection. + ``node_reports`` holds each connected node supervisor's latest report. Also update + ``stalled_replacements`` and, when a change ends, ``last_transaction``. + """ + before, committed = self.record.state, self._committed_mask + paused_s: dict[int, float] = {} + for node_report in node_reports: + received_s = node_report.received_s + if now_s - received_s <= self.hang_timeout_s / 2: + for r in _ranks(node_report.paused_mask): + paused_s[r] = max(paused_s.get(r, received_s), received_s) + record = self._decide(now_s, reports, connected, paused_s) + if before != MembershipState.STARTING: + self.stalled_replacements = tuple( + r + for r in sorted(self._requested) + if not reports.get(r, RankReport()).joining_ready + and not self._startup_advancing(r, reports, now_s) + ) + if record is None: + return None + if before == MembershipState.COMMITTED: + self._transaction_began_s = now_s + if before != MembershipState.STARTING and record.state in ( + MembershipState.COMMITTED, + MembershipState.ABORTED, + ): + self._record_transaction(committed, record, now_s) + return record + + def _record_transaction(self, committed: int, record: MembershipRecord, now_s: float) -> None: + """Count the change that ``record`` ends and summarize it in ``last_transaction``.""" + now_unix_ms = time.time() * 1e3 + + def unix_ms(s: Optional[float]) -> Optional[float]: + return None if s is None else now_unix_ms + (s - now_s) * 1e3 + + began_s = self._transaction_began_s + removed = tuple(_ranks(committed & ~record.active_mask)) + if record.state == MembershipState.ABORTED: + kind = "abort" + else: + kind = "failover" if removed else "admission" + self._changes[kind] = self._changes.get(kind, 0) + 1 + self.last_transaction = TransactionSummary( + generation=record.generation, + outcome=record.state.name, + removed_ranks=removed, + causes={r: self._causes[r] for r in removed if r in self._causes}, + added_ranks=tuple(_ranks(record.active_mask & ~committed)), + detected_unix_ms=unix_ms(began_s if self._detected_s is None else self._detected_s), + began_unix_ms=unix_ms(began_s), + ready_unix_ms=unix_ms(self._ready_s), + ended_unix_ms=now_unix_ms, + changes=dict(self._changes), + reason=record.reason if record.state == MembershipState.ABORTED else "", + ) + self._detected_s = None + self._causes = {} + + def _decide( + self, + now_s: float, + reports: dict[int, RankReport], + connected: int, + paused_s: dict[int, float], + ) -> Optional[MembershipRecord]: + """Return the record that follows from the reports, or None if membership stays. + + ``paused_s`` maps each rank whose hang detector its node supervisor recently reported + paused to when that report arrived. + """ + state = self.record.state + if state in (MembershipState.ABORTED, MembershipState.SHUTDOWN): + return None + if 0 in reports and reports[0].shutdown: + return self._new_record(MembershipState.SHUTDOWN, self.record.active_mask) + if state == MembershipState.STARTING: + return self._await_startup(now_s, reports, connected) + failed = self._detect_failures(now_s, reports, connected, self.hang_timeout_s, paused_s) + # A replacement that disconnects or fails withdraws its join request. + self._forget_joins((failed | ~connected) & all_ranks(self.world_size)) + if state == MembershipState.COMMITTED: + return self._start_change(now_s, reports, failed) + if failed: + return self._propose(self.record.active_mask & ~failed) + return self._advance_change(now_s, reports) + + def _forget_joins(self, gone_mask: int) -> None: + gone = set(_ranks(gone_mask)) + self._requested -= gone + self._joining -= gone + + def _start_change( + self, now_s: float, reports: dict[int, RankReport], failed: int + ) -> Optional[MembershipRecord]: + """Propose to remove the ``failed`` ranks and to add the replacements ready to join.""" + ready = {r for r in self._requested if r in reports and reports[r].joining_ready} + starting = {r for r in self._requested if self._startup_advancing(r, reports, now_s)} + if not failed and not ready: + self._detected_s = None + return None + if self._detected_s is None: + self._detected_s = now_s + if not failed and self._waits_for_replacements_of_same_failure( + now_s, reports, ready, starting + ): + return None + self._requested -= ready + self._joining |= ready + if ready: + logger.info(f"Adding joining ranks {sorted(ready)} to the membership") + return self._propose( + (self.record.active_mask & ~failed) | rank_mask(ready, self.world_size) + ) + + def _advance_change( + self, now_s: float, reports: dict[int, RankReport] + ) -> Optional[MembershipRecord]: + """Move a PROPOSE to INSTALL, or an INSTALL to COMMITTED, once every active rank replied. + + A survivor that replies FAILED aborts the deployment. A joining rank that replies FAILED is + left out. + """ + state = self.record.state + active = _ranks(self.record.active_mask) + failed_replies = {r for r in active if self._rank_replied(r, reports, Reply.FAILED)} + failed_survivors = failed_replies - self._joining + if failed_survivors: + return self.abort(f"in {state.name}, ranks {sorted(failed_survivors)} replied FAILED") + if failed_replies: + self._joining -= failed_replies + return self._propose( + self.record.active_mask & ~rank_mask(failed_replies, self.world_size), + reason="a replacement could not install the change that adds it", + ) + expected = Reply.READY if state == MembershipState.PROPOSE else Reply.INSTALLED + if not all(self._rank_replied(r, reports, expected) for r in active): + return None + if state == MembershipState.PROPOSE: + return self._install(now_s, reports, active) + return self._commit(active) + + def _rank_replied(self, r: int, reports: dict[int, RankReport], reply: Reply) -> bool: + """Return whether rank ``r`` replied ``reply``, counting a joiner's readiness as READY.""" + report = reports.get(r) + ready_to_join = r in self._joining and report is not None and report.joining_ready + return (reply == Reply.READY and ready_to_join) or self._replied(report, reply) + + def _install( + self, now_s: float, reports: dict[int, RankReport], active: list[int] + ) -> MembershipRecord: + """Publish INSTALL with the point where every active rank resumes.""" + survivors = [ + r for r in active if has_rank(self._committed_mask, r) and r not in self._joining + ] + # Resume after the furthest survivor, so no rank repeats an iteration that another finished. + resume = ResumeState( + iteration=max( + [self.record.resume.iteration] + [reports[r].paused_at.iteration for r in survivors] + ), + epoch=self._resume_epoch([reports[r].paused_at.epoch for r in survivors]), + ) + self._ready_s = now_s + return self._new_record(MembershipState.INSTALL, self.record.active_mask, resume) + + def _resume_epoch(self, epochs: list[int]) -> int: + """Return the host exchange epoch to resume from, given the survivors' paused epochs.""" + replaced = self._replaced_install_epoch + # A newer PROPOSE can replace an INSTALL before its commit, after some ranks moved to that + # INSTALL's resume epoch. Every round waits for every active rank, so if a survivor is still + # before that epoch, no rank ran a round at it. Resume there again, which keeps the ranks + # that installed it consistent. + if replaced and any(latest_epoch([replaced, epoch]) != epoch for epoch in epochs): + return replaced + # Epochs wrap, so the resume epoch follows the latest one, not the largest. + return next_epoch(latest_epoch(epochs)) + + def _commit(self, active: list[int]) -> MembershipRecord: + """Commit the change that every active rank installed.""" + record = self.record + self._joining -= set(active) + self._replaced_install_epoch = 0 + for r in _ranks(self._committed_mask & ~record.active_mask): + self._failed_in[r] = record.generation + for r in active: + self._failed_in.pop(r, None) + self._committed_mask = record.active_mask + # A replacement's progress clock must not continue from the process it replaces. + self._progress.clear() + self._hung_since.clear() + return self._new_record(MembershipState.COMMITTED, record.active_mask) + + def _await_startup( + self, now_s: float, reports: dict[int, RankReport], connected: int + ) -> Optional[MembershipRecord]: + """Commit generation 0 once every rank connected and started. + + Abort if a connected rank fails, or if a rank does not connect within the rendezvous + timeout. + """ + if self._rendezvous_began_s is None: + self._rendezvous_began_s = now_s + everyone = self.record.active_mask + self._connected_once |= connected + failed = self._detect_failures(now_s, reports, connected, self.rendezvous_timeout_s, {}) + if failed & self._connected_once: + return self.abort( + f"ranks {_ranks(failed & self._connected_once)} failed during the startup" + ) + absent = everyone & ~self._connected_once + if absent and now_s - self._rendezvous_began_s > self.rendezvous_timeout_s: + return self.abort( + f"ranks {_ranks(absent)} did not reach the rendezvous within " + f"{self.rendezvous_timeout_s:.0f} s" + ) + if absent or not all(reports.get(r, RankReport()).started for r in _ranks(everyone)): + return None + logger.info("Every rank started") + self._progress.clear() + self._hung_since.clear() + self._detected_s = None + self._causes = {} + return self._new_record(MembershipState.COMMITTED, everyone) + + def _waits_for_replacements_of_same_failure( + self, now_s: float, reports: dict[int, RankReport], ready: set[int], starting: set[int] + ) -> bool: + """Return whether to delay adding ``ready`` for a replacement that is still starting. + + Delay while a ``starting`` replacement of a rank that one change removed together with a + rank of ``ready`` still reports. The replacements of a lost node then join in one change, + and the survivors pause once. + """ + failed_with = {self._failed_in[r] for r in ready if r in self._failed_in} + return any( + self._failed_in.get(r) in failed_with and self._reports_advance(r, reports, now_s) + for r in starting - ready + ) + + def _startup_advancing(self, r: int, reports: dict[int, RankReport], now_s: float) -> bool: + """Return whether the startup of rank ``r``'s replacement progressed within the timeout.""" + report = reports.get(r, RankReport()) + progress = (report.incarnation, report.progress) + # A replacement waits for its predecessor's memory until the active ranks commit a + # membership without the predecessor. This wait is not a stall. + if has_rank(self._committed_mask, r): + self._startup_progress.pop(r, None) + changed_s = _update_last_change(self._startup_progress, r, progress, now_s) + return now_s - changed_s <= self.rendezvous_timeout_s + + def _reports_advance(self, r: int, reports: dict[int, RankReport], now_s: float) -> bool: + """Return whether rank ``r`` sent a report within half the hang timeout. + + A rank whose reports stopped is a stopped process, so its last ``waiting_for`` is stale. + """ + return now_s - self._received_s(r, reports, now_s) <= self.hang_timeout_s / 2 + + def _received_s(self, r: int, reports: dict[int, RankReport], now_s: float) -> float: + """Return when rank ``r``'s latest report arrived, or since when it has sent none.""" + if r in reports: + self._unreported_since.pop(r, None) + return reports[r].received_s + return self._unreported_since.setdefault(r, now_s) + + def _propose(self, active_mask: int, reason: str = "") -> MembershipRecord: + """Publish PROPOSE of ``active_mask`` in a new generation, or ABORTED without rank 0.""" + self._ready_s = None + # Rank 0 hosts the authority and serves the requests, so the deployment ends without it. + if not has_rank(active_mask, 0): + logger.error("Deployment aborted: rank 0 failed") + return self._new_record(MembershipState.ABORTED, active_mask, reason="rank 0 failed") + if self.record.state == MembershipState.INSTALL: + self._replaced_install_epoch = self.record.resume.epoch + return self._new_record( + MembershipState.PROPOSE, + active_mask, + generation=self.record.generation + 1, + reason=reason, + ) + + def abort(self, reason: str) -> MembershipRecord: + """Publish ABORTED with ``reason``, also for a fatal error outside the reports.""" + logger.error(f"Deployment aborted: {reason}") + return self._new_record(MembershipState.ABORTED, self.record.active_mask, reason=reason) + + def _new_record( + self, + state: MembershipState, + active_mask: int, + resume: Optional[ResumeState] = None, + generation: Optional[int] = None, + reason: str = "", + ) -> MembershipRecord: + """Replace ``record`` with one in ``state``, keeping the generation and resume unless given.""" + self.record = MembershipRecord( + generation=self.record.generation if generation is None else generation, + state=state, + active_mask=active_mask, + resume=self.record.resume if resume is None else resume, + joining_mask=rank_mask(self._joining, self.world_size) & active_mask, + reason=reason, + ) + return self.record + + def _replied(self, report: Optional[RankReport], reply: Reply) -> bool: + return ( + report is not None + and report.reply_generation == self.record.generation + and report.reply == reply + ) + + def _detect_failures( + self, + now_s: float, + reports: dict[int, RankReport], + connected: int, + timeout_s: float, + paused_s: dict[int, float], + ) -> int: + """Return a mask of active ranks that are dead, or hung for ``timeout_s``. + + A rank whose connection closed is dead. A rank is hung when other ranks wait on it, it is + the end of a wait-for chain (:func:`_chain_ends`), and it made no progress for the timeout. + Progress counts its iteration, install steps, and the all-to-all rounds and host + collective epochs its peers wait on. During a change the ranks that have not replied are + waited on, and during startup the ranks that have not started. A rank's waits count only + while its reports advance. Outside a change, a rank in ``paused_s``, whose node supervisor + recently reported its hang detector paused, is never hung, and once it leaves the pause its + stall starts no earlier than the latest node report that counted it as paused. A joining + rank that waits on no peer is hung after at most the admission timeout, so the survivors do + not pause a full hang timeout for it. + + Also record the failure causes and since when the active ranks stall. + """ + state, active_mask = self.record.state, self.record.active_mask + active = _ranks(active_mask) + expected = {MembershipState.PROPOSE: Reply.READY, MembershipState.INSTALL: Reply.INSTALLED} + waited_on = 0 + if state in expected: + waited_on = rank_mask( + (r for r in active if not self._replied(reports.get(r), expected[state])), + self.world_size, + ) + # During the startup a rank that started waits on every rank that has not. + not_started = 0 + if state == MembershipState.STARTING: + not_started = rank_mask( + (r for r in active if not reports.get(r, RankReport()).started), self.world_size + ) + # A rank's waits count only while its reports advance. + waits: dict[int, int] = {} + rounds: dict[int, set[tuple]] = {} + for r in active: + report = reports.get(r, RankReport()) + if self._reports_advance(r, reports, now_s): + waiting_for = report.waiting_for | (not_started if report.started else 0) + waits[r] = waiting_for & active_mask & ~(1 << r) + waited_on |= waits[r] + for peer, round_flag in report.waited_flags: + rounds.setdefault(peer, set()).add(("all-to-all", round_flag)) + for peer, epoch in report.waited_epochs: + rounds.setdefault(peer, set()).add(("host exchange", epoch)) + chain_ends = _chain_ends(active, waits) + failed = active_mask & ~connected + for r in _ranks(failed): + self._causes.setdefault(r, "disconnected") + self.stalled_since_s = None + for r in active: + report = reports.get(r, RankReport()) + progress = (report.progress, report.install_steps, frozenset(rounds.get(r, ()))) + progressed_s = _update_last_change(self._progress, r, progress, now_s) + # A rank in a call that its hang detector's pause covers, such as rank 0 waiting for + # requests, keeps the others waiting but is not hung, also once its reports stop: the + # call may hold the GIL that its membership client thread needs. Its node supervisor + # reads the pause from shared memory, so a stopped rank or one that left the pause is + # not taken as paused. During a change every rank pauses its hang detector, so the + # pause counts only outside a change. + paused = state == MembershipState.COMMITTED and r in paused_s + if paused: + self._last_paused_s[r] = paused_s[r] + if has_rank(waited_on, r) and not paused and has_rank(chain_ends, r): + # A rank whose reports stopped advancing has stalled since its last report, or + # since it left a pause, whichever is later. + stalled_s = now_s if r in waits else self._received_s(r, reports, now_s) + since_s = max( + self._hung_since.setdefault(r, stalled_s), + progressed_s, + self._last_paused_s.get(r, progressed_s), + ) + # Publish a stall only once it outlasts a report interval. + if now_s - since_s > REPORT_INTERVAL_S and ( + self.stalled_since_s is None or since_s < self.stalled_since_s + ): + self.stalled_since_s = since_s + # A joiner that waits on a peer may be held up by it, so it gets the full timeout. + stalled_joiner = r in self._joining and not waits.get(r) + limit_s = min(timeout_s, self.admission_timeout_s) if stalled_joiner else timeout_s + if now_s - since_s > limit_s: + failed |= 1 << r + self._causes.setdefault(r, "hung") + if self._detected_s is None or since_s < self._detected_s: + self._detected_s = since_s + logger.warning( + f"Rank {r} is hung. It made no progress for {now_s - since_s:.1f} s while " + "active ranks wait for it. Last reports: " + f"{self._describe(active, reports, waits, now_s, paused_s)}" + ) + else: + self._hung_since.pop(r, None) + return failed + + def _describe( + self, + active: list[int], + reports: dict[int, RankReport], + waits: dict[int, int], + now_s: float, + paused_s: dict[int, float], + ) -> str: + described = [] + for r in active: + report = reports.get(r, RankReport()) + described.append( + f"rank {r}: iteration {report.progress}, paused {r in paused_s}, " + f"waits for {_ranks(waits.get(r, 0))}, " + f"all-to-all rounds {list(report.waited_flags)}, " + f"host-exchange epochs {list(report.waited_epochs)}, reported " + f"{now_s - self._received_s(r, reports, now_s):.1f} s ago" + ) + return "; ".join(described) diff --git a/tensorrt_llm/_torch/distributed/membership/authority_process.py b/tensorrt_llm/_torch/distributed/membership/authority_process.py new file mode 100644 index 000000000000..6fa456c37d2d --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/authority_process.py @@ -0,0 +1,654 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Membership authority process for Wide-EP fault tolerance. + +:class:`AuthorityProcess` runs an :class:`~.authority.Authority`, serves the liveness connections +from ranks and node supervisors, and publishes each new record in the rendezvous store. +""" + +import argparse +import hmac +import logging +import logging.handlers +import os +import queue +import selectors +import signal +import socket +import sys +import time +from dataclasses import dataclass +from typing import Optional + +from tensorrt_llm._wide_ep_fault_tolerance import RENDEZVOUS_ENV, SHARED_SECRET_ENV +from tensorrt_llm.logger import logger + +from .authority import Authority +from .liveness import Lines, configure_connection, hello_token +from .records import ( + HEARTBEAT_INTERVAL_S, + HELLO_TIMEOUT_S, + MAX_RANKS, + NODE_SUPERVISOR_TIMEOUT_S, + REPORT_INTERVAL_S, + MembershipRecord, + MembershipState, + NodeReport, + NodeView, + RankReport, + all_ranks, + has_rank, + message_bytes, + rank_mask, + remove_cpu_affinity_constraints, +) +from .rendezvous import ( + LIVENESS_KEY, + RECORD_KEY, + STALLED_SINCE_KEY, + TRANSACTION_KEY, + rendezvous_address, + rendezvous_host_port, +) +from .signed_store import SignedStore + +# A peer with more unsent bytes queued than this reads nothing, so its connection is closed. +_MAX_UNSENT_BYTES = 1 << 22 +# Selector data for rank 0's pipe. End of file on it means rank 0's process ended; any bytes +# written to it are read and ignored. +_PARENT = object() + + +def _decided_at_once(report: RankReport) -> tuple: + """Return the report fields that the authority acts on as soon as they change.""" + return ( + report.reply_generation, + report.reply, + report.joining_ready, + report.started, + report.shutdown, + ) + + +@dataclass +class _Connection: + """A liveness connection the authority accepted.""" + + accepted_s: float + sent_s: float = 0.0 + received: bytes = b"" + member: Optional[tuple[int, int]] = None # (rank, incarnation) + node: Optional[int] = None + node_ranks: tuple[int, ...] = () + lines: Optional[Lines] = None + report: Optional[RankReport] = None + node_report: Optional[NodeReport] = None + # The node report is older than half the hang timeout, so the authority ignores it. + node_report_stale: bool = False + unsent: bytes = b"" + + +class AuthorityProcess: + """Run :class:`Authority` and publish every record in the store and on every connection. + + A replacement is admitted only for a rank that the membership no longer lists and that has no + connected process, so only the authority's verdict removes a process, and each rank has at most + one connection. The deployment aborts once ``parent``, held by rank 0's process, ends. + """ + + def __init__( + self, + world_size: int, + gpus_per_node: int, + store: SignedStore, + authority: Authority, + parent: Optional[int] = None, + ) -> None: + """Prepare to serve. :meth:`run` listens and publishes the first record. + + Args: + world_size: The deployment's ranks. + gpus_per_node: The ranks per node, by which a missing node supervisor is named. + store: The rendezvous store, which this process hosts. + authority: The authority that decides the membership. + parent: The read end of a pipe whose write end rank 0's process holds, or None. + """ + self.world_size = world_size + self._gpus_per_node = gpus_per_node + self._store = store + self._authority = authority + self._hello_token = hello_token().encode() + self._parent = parent + self._parent_ended_s: Optional[float] = None + self._changed = False + self._incarnation = [0] * world_size + self._connections: dict[socket.socket, _Connection] = {} + self._selector = selectors.DefaultSelector() + self._record: Optional[MembershipRecord] = None + # Ranks whose node supervisor connected, and when the authority started. + self._supervised_mask = 0 + self._started_s = 0.0 + self._sent_stalled_replacements: tuple[int, ...] = () + self._sent_connected_mask = 0 + self._published_stalled_since_s: Optional[float] = None + # The ranks that no replacement is started for, which every published record carries. + self._given_up_mask = 0 + + def run(self, host: str) -> MembershipRecord: + """Serve on ``host`` until the record is ABORTED or SHUTDOWN, and return that record.""" + try: + with socket.create_server((host, 0)) as listener: + listener.setblocking(False) + self._selector.register(listener, selectors.EVENT_READ) + if self._parent is not None: + self._selector.register(self._parent, selectors.EVENT_READ, _PARENT) + self._publish(self._authority.record) + self._started_s = time.monotonic() + self._store.set( + LIVENESS_KEY, + f"{host}:{listener.getsockname()[1]}".encode(), + ) + try: + return self._serve() + except Exception as error: + # The store ends with this process, so the ranks learn why from their + # connections. + record = self._authority.abort(f"the authority failed: {error!r}") + self._record = self._stamped(record) + self._send_record() + raise + finally: + for sock in list(self._connections): + sock.close() + self._selector.close() + + def _serve(self) -> MembershipRecord: + decide_s = time.monotonic() + while True: + self._poll(max(0.0, decide_s - time.monotonic())) + now_s = time.monotonic() + if not self._changed and now_s < decide_s: + continue + if now_s - decide_s > HEARTBEAT_INTERVAL_S: + logger.warning( + f"The membership authority decided {now_s - decide_s:.1f} s late, so its " + "heartbeats to the ranks were late too" + ) + self._changed = False + decide_s = now_s + REPORT_INTERVAL_S + self._send_heartbeats(time.monotonic()) + summary = self._authority.last_transaction + if self._parent_ended_s is not None: + record = self._authority.abort("rank 0's process ended") + else: + record = self._check_supervisors(time.monotonic()) + connected = self._connected() + if record is None: + tick_s = time.monotonic() + record = self._authority.tick( + tick_s, self._reports(), connected, self._node_reports(tick_s) + ) + self._publish_stall() + if record is None: + if self._record.given_up_mask != self._given_up_mask: + # Resend the record to the ranks only once committed. During a change a rank + # would take a resent record as the change's next step. + self._publish( + self._record, ranks=self._record.state == MembershipState.COMMITTED + ) + elif ( + self._authority.stalled_replacements != self._sent_stalled_replacements + or connected != self._sent_connected_mask + ): + self._send_record(ranks=False) + continue + logger.info(f"Membership: {record!r}") + transaction = self._authority.last_transaction + if transaction is not summary: + logger.info(f"Membership transaction: {transaction!r}") + self._store.set(TRANSACTION_KEY, transaction.model_dump_json().encode()) + record = self._publish(record) + if record.state in (MembershipState.ABORTED, MembershipState.SHUTDOWN): + # The store lives in this process, so keep it until every active rank has read the + # record and left, for at most a hang timeout after an abort or rank 0's exit. + aborted_s = time.monotonic() if record.state == MembershipState.ABORTED else None + while self._connected() & record.active_mask: + since_s = self._parent_ended_s if aborted_s is None else aborted_s + if ( + since_s is not None + and time.monotonic() - since_s >= self._authority.hang_timeout_s + ): + break + # Ranks that have not left yet take a silent authority as hung. + self._send_heartbeats(time.monotonic()) + self._poll(REPORT_INTERVAL_S) + return record + + def _check_supervisors(self, now_s: float) -> Optional[MembershipRecord]: + """Abort a startup in which some nodes' supervisors did not connect within the timeout. + + Return the ABORTED record, or None if the startup goes on. + """ + unsupervised = all_ranks(self.world_size) & ~self._supervised_mask + if ( + self._authority.record.state != MembershipState.STARTING + or not unsupervised + or now_s - self._started_s < NODE_SUPERVISOR_TIMEOUT_S + ): + return None + nodes = sorted( + { + rank // self._gpus_per_node + for rank in range(self.world_size) + if has_rank(unsupervised, rank) + } + ) + return self._authority.abort( + f"the supervisors of nodes {nodes} did not connect within " + f"{NODE_SUPERVISOR_TIMEOUT_S:.0f} s (one per node, with {SHARED_SECRET_ENV} and " + f"{RENDEZVOUS_ENV} set)" + ) + + def _stamped(self, record: MembershipRecord) -> MembershipRecord: + """Return ``record`` with the current incarnations and given-up ranks.""" + return record.model_copy( + update={"incarnations": tuple(self._incarnation), "given_up_mask": self._given_up_mask} + ) + + def _publish(self, record: MembershipRecord, ranks: bool = True) -> MembershipRecord: + record = self._stamped(record) + self._store.set(RECORD_KEY, record.model_dump_json().encode()) + self._record = record + self._send_record(ranks) + return record + + def _publish_stall(self) -> None: + """Publish when the active ranks started to wait on a stalled rank, in Unix milliseconds.""" + stalled_since_s = self._authority.stalled_since_s + if stalled_since_s == self._published_stalled_since_s: + return + self._published_stalled_since_s = stalled_since_s + if stalled_since_s is None: + value = b"" + else: + stalled_for_s = time.monotonic() - stalled_since_s + value = str(round((time.time() - stalled_for_s) * 1e3)).encode() + self._store.set(STALLED_SINCE_KEY, value) + + def _send_record(self, ranks: bool = True) -> None: + record = message_bytes(self._record) + self._sent_stalled_replacements = self._authority.stalled_replacements + self._sent_connected_mask = self._connected() + view = message_bytes( + NodeView( + record=self._record, + stalled_replacements=self._sent_stalled_replacements, + connected_mask=self._sent_connected_mask, + ) + ) + for sock, connection in list(self._connections.items()): + # A connection an earlier send in this loop dropped is gone. + if sock in self._connections and connection.lines is not None: + if connection.node is not None or ranks: + self._send(sock, connection, record if connection.node is None else view) + + def _send_heartbeats(self, now_s: float) -> None: + """Send an empty line to each rank that has received nothing for the heartbeat interval.""" + for sock, connection in list(self._connections.items()): + if ( + sock in self._connections + and connection.member is not None + and now_s - connection.sent_s >= HEARTBEAT_INTERVAL_S + ): + self._send(sock, connection, b"") + + def _send(self, sock: socket.socket, connection: _Connection, payload: bytes) -> None: + connection.unsent += connection.lines.encode(payload) + connection.sent_s = time.monotonic() + self._flush(sock, connection) + + def _flush(self, sock: socket.socket, connection: _Connection) -> None: + try: + sent = sock.send(connection.unsent) + except BlockingIOError: + sent = 0 + except OSError: + self._drop(sock) + return + connection.unsent = connection.unsent[sent:] + if len(connection.unsent) > _MAX_UNSENT_BYTES: + if connection.node is None: + logger.warning( + f"Closing the connection of {connection.member}, which reads nothing" + ) + self._drop(sock, problem="it reads nothing") + return + write = selectors.EVENT_WRITE if connection.unsent else 0 + self._selector.modify(sock, selectors.EVENT_READ | write, connection) + + def _poll(self, timeout_s: float) -> None: + for key, events in self._selector.select(timeout_s): + sock, connection = key.fileobj, key.data + if connection is None: + self._accept(sock) + continue + if connection is _PARENT: + if not os.read(sock, 1 << 12): + self._selector.unregister(sock) + self._parent_ended_s = time.monotonic() + self._changed = True + continue + if sock not in self._connections: + continue + if events & selectors.EVENT_WRITE: + self._flush(sock, connection) + if events & selectors.EVENT_READ and sock in self._connections: + self._receive(sock, connection) + now_s = time.monotonic() + for sock, connection in list(self._connections.items()): + if ( + sock in self._connections + and connection.lines is None + and now_s - connection.accepted_s > HELLO_TIMEOUT_S + ): + logger.warning("Closing a liveness connection without a valid hello") + self._drop(sock) + + def _accept(self, listener: socket.socket) -> None: + """Accept a connection, dropping the oldest of ``MAX_RANKS`` that wait for their hello.""" + waiting = [sock for sock, c in self._connections.items() if c.lines is None] + if len(waiting) >= MAX_RANKS: + logger.warning("Closing the oldest liveness connection waiting for its hello") + self._drop(waiting.pop(0)) + try: + sock, _ = listener.accept() + except BlockingIOError: + return + except OSError as error: # out of descriptors (EMFILE, ENFILE) or memory + logger.warning(f"Cannot accept a liveness connection: {error!r}") + if waiting: + self._drop(waiting[0]) + return + sock.setblocking(False) + configure_connection(sock) + self._connections[sock] = _Connection(time.monotonic()) + self._selector.register(sock, selectors.EVENT_READ, self._connections[sock]) + + def _receive(self, sock: socket.socket, connection: _Connection) -> None: + try: + data = sock.recv(1 << 16) + except BlockingIOError: + return + except OSError: + data = b"" + if not data: + self._drop(sock) + return + try: + if connection.lines is None: + data = self._read_hello(sock, connection, data) + # The connection is gone if sending the first record failed. + if sock not in self._connections: + return + for line in connection.lines.decode(data) if data else (): + if connection.node is not None: + self._on_node_report(connection, NodeReport.model_validate_json(line)) + continue + if connection.member is None: + raise ValueError("a report arrived on a liveness connection of no rank") + report = RankReport.model_validate_json(line).model_copy( + update={"incarnation": connection.member[1], "received_s": time.monotonic()} + ) + if connection.report is None or _decided_at_once( + connection.report + ) != _decided_at_once(report): + self._changed = True + connection.report = report + except ValueError as error: + if connection.node is None: + logger.warning(f"Closing a liveness connection: {error}") + self._drop(sock, problem=f"it sent an invalid line: {error}") + + def _read_hello(self, sock: socket.socket, connection: _Connection, data: bytes) -> bytes: + """Admit the peer whose hello line ``data`` completes and return the bytes after it. + + A rank sends `` <1 if a replacement, else 0>`` and a node supervisor sends + `` node ``. + """ + connection.received += data + if b"\n" not in connection.received: + if len(connection.received) > 256: + raise ValueError("no hello") + return b"" + line, rest = connection.received.split(b"\n", 1) + token, *announcement = line.split() + if not hmac.compare_digest(token, self._hello_token): + raise ValueError("the hello does not carry this deployment's token") + announcement = [field.decode() for field in announcement] + if announcement[:1] == ["node"]: + _, node, ranks, host = announcement + node, ranks = int(node), tuple(int(rank) for rank in ranks.split(",")) + refusal = self._node_refusal(node, ranks) + if refusal is not None: + raise ValueError(refusal) + connection.node, connection.node_ranks, connection.lines = node, ranks, Lines() + self._supervised_mask |= rank_mask(ranks, self.world_size) + logger.info(f"The supervisor of node {node} ({host}, ranks {list(ranks)}) connected") + self._send_record(ranks=False) + return rest + rank, replacement = (int(field) for field in announcement) + refusal = self._refusal(rank, replacement) + if refusal is not None: + raise ValueError(refusal) + if replacement: + self._incarnation[rank] += 1 + self._authority.request_join(rank) + connection.member = (rank, self._incarnation[rank]) + self._changed = True + connection.lines = Lines() + record = self._record.model_copy(update={"incarnations": tuple(self._incarnation)}) + self._send(sock, connection, message_bytes(record)) + return rest + + def _on_node_report(self, connection: _Connection, report: NodeReport) -> None: + """Keep ``report`` from the supervisor on ``connection`` and take the ranks it gave up.""" + if not set(report.given_up) <= set(connection.node_ranks): + raise ValueError( + f"node {connection.node} gave up ranks {list(report.given_up)}, which are not all " + "its own" + ) + node_mask = rank_mask(connection.node_ranks, self.world_size) + if report.paused_mask & ~node_mask: + raise ValueError( + f"node {connection.node} reported the ranks of mask {report.paused_mask:#x} " + "paused, which are not all its own" + ) + connection.node_report = report.model_copy(update={"received_s": time.monotonic()}) + given_up_mask = (self._given_up_mask & ~node_mask) | rank_mask( + report.given_up, self.world_size + ) + # The supervisor reports every report interval, and only a new given-up rank is decided + # at once. + if given_up_mask != self._given_up_mask: + self._given_up_mask = given_up_mask + self._changed = True + + def _node_refusal(self, node: int, ranks: tuple[int, ...]) -> Optional[str]: + if not ranks or not all(0 <= rank < self.world_size for rank in ranks): + return f"node {node}'s ranks {list(ranks)} are not all in the deployment" + for c in self._connections.values(): + if c.node == node or set(c.node_ranks) & set(ranks): + return f"node {node} or one of its ranks {list(ranks)} has a supervisor already" + return None + + def _refusal(self, rank: int, replacement: int) -> Optional[str]: + """Return why a hello from ``rank`` is refused, or None to admit it. + + A replacement is admitted only once the membership no longer lists the rank, so its + connection never hides that its predecessor ended. The startup's record lists every rank, + so no replacement joins before the deployment has started. + """ + if not 0 <= rank < self.world_size: + return f"rank {rank} is not in the deployment" + if replacement: + if rank == 0: + return "rank 0 is never replaced" + if self._authority.record.is_active(rank): + return f"rank {rank} is active: no replacement joins for it" + elif self._authority.record.state != MembershipState.STARTING: + return f"rank {rank} joins only as a replacement once the deployment has started" + # A first process and a replacement alike join only while the rank has no connected process. + if any(c.member is not None and c.member[0] == rank for c in self._connections.values()): + return f"rank {rank} has a process connected already" + return None + + def _drop(self, sock: socket.socket, problem: str = "") -> None: + """Close ``sock``, and fail the node's ranks if it is a supervisor's connection. + + No replacement for those ranks starts until their node's supervisor connects again, so the + authority publishes them as given up. + + Args: + problem: Why the authority closes the connection, or empty if the peer closed it. + """ + if sock not in self._connections: + return + self._selector.unregister(sock) + connection = self._connections.pop(sock) + sock.close() + self._changed = True + if connection.node is None: + return + if self._record.state not in (MembershipState.ABORTED, MembershipState.SHUTDOWN): + ranks = list(connection.node_ranks) + if problem: + logger.error( + f"Closing the connection of the supervisor of node {connection.node}, " + f"because {problem}. Its ranks {ranks} count as failed" + ) + else: + logger.warning( + f"The supervisor of node {connection.node} left, " + f"so its ranks {ranks} count as failed" + ) + self._given_up_mask |= rank_mask(ranks, self.world_size) + for other, peer in list(self._connections.items()): + if peer.member is not None and peer.member[0] in connection.node_ranks: + self._drop(other) + self._send_record(ranks=False) + + def _connected(self) -> int: + return rank_mask( + (c.member[0] for c in self._connections.values() if c.member is not None), + self.world_size, + ) + + def _reports(self) -> dict[int, RankReport]: + return {c.member[0]: c.report for c in self._connections.values() if c.report is not None} + + def _node_reports(self, now_s: float) -> list[NodeReport]: + """Return each node supervisor's latest report, and log when one goes stale or resumes. + + The authority ignores a report older than half the hang timeout, so the node's paused ranks + lose their exemption from the hang rule until the supervisor reports again. + """ + stale_s = self._authority.hang_timeout_s / 2 + reports = [] + for c in self._connections.values(): + if c.node_report is None: + continue + stale = now_s - c.node_report.received_s > stale_s + if stale != c.node_report_stale: + c.node_report_stale = stale + if stale: + logger.warning( + f"The supervisor of node {c.node} has not reported for {stale_s:g} s, so " + "its paused ranks are no longer exempt from the hang rule" + ) + else: + logger.info( + f"The supervisor of node {c.node} reports again, so its paused ranks are " + "exempt from the hang rule again" + ) + reports.append(c.node_report) + return reports + + +def log_from_own_thread() -> logging.handlers.QueueListener: + """Write this process's log records from their own thread, and return its listener. + + The authority shares rank 0's standard output, whose writes can block. A blocked write must not + hold up the heartbeats. + """ + python_logger = logging.getLogger(logger.PREFIX) + records = queue.SimpleQueue() + listener = logging.handlers.QueueListener( + records, *python_logger.handlers, respect_handler_level=True + ) + python_logger.handlers = [logging.handlers.QueueHandler(records)] + listener.start() + return listener + + +def main(argv: Optional[list[str]] = None) -> int: + """Run the authority process. + + Start the rendezvous store at the address in ``TLLM_FAULT_TOLERANCE_RENDEZVOUS``, then write + that address and a newline to ``--address-fd`` to tell the parent that the store is up. Abort + the deployment once rank 0's process closes the standard input. + + Args: + argv: The command-line arguments, by default ``sys.argv[1:]``. + + Returns: + The exit status: 0 once the deployment shut down, 1 once it aborted. + """ + parser = argparse.ArgumentParser(description="Wide-EP fault-tolerance membership authority") + parser.add_argument("--world-size", type=int, required=True) + parser.add_argument("--gpus-per-node", type=int, required=True) + parser.add_argument("--address-fd", type=int, required=True) + args = parser.parse_args(argv) + remove_cpu_affinity_constraints() + log_listener = log_from_own_thread() + + from torch.distributed import TCPStore + + address = rendezvous_address() + host, port = rendezvous_host_port(address) + # Like the liveness listener, the store listens only on the address the deployment uses. + listener = socket.create_server((host, port)) + store = SignedStore( + TCPStore( + host, + port, + is_master=True, + wait_for_workers=False, + master_listen_fd=listener.detach(), + ) + ) + os.write(args.address_fd, f"{address}\n".encode()) + os.close(args.address_fd) + authority = Authority(args.world_size) + # A signal for the deployment (Ctrl-C in its terminal) is its ranks' to act on. + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, signal.SIG_IGN) + process = AuthorityProcess( + args.world_size, args.gpus_per_node, store, authority, parent=sys.stdin.fileno() + ) + try: + record = process.run(host) + finally: + log_listener.stop() + return 0 if record.state == MembershipState.SHUTDOWN else 1 diff --git a/tensorrt_llm/_torch/distributed/membership/liveness.py b/tensorrt_llm/_torch/distributed/membership/liveness.py new file mode 100644 index 000000000000..4d9f870350b6 --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/liveness.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Liveness connections from Wide-EP ranks and node supervisors to the membership authority.""" + +import functools +import os +import select +import socket +import threading +import time +import traceback +import weakref +from collections.abc import Callable +from dataclasses import dataclass +from datetime import timedelta +from typing import IO, Optional + +from tensorrt_llm.logger import logger + +from .records import ( + AUTHORITY_SILENCE_TIMEOUT_S, + DEAD_TIMEOUT_S, + HELLO_TIMEOUT_S, + REPORT_INTERVAL_S, + MembershipRecord, + RankReport, + message_bytes, +) +from .rendezvous import LIVENESS_KEY, rendezvous_address, rendezvous_host_port +from .signed_store import SignedStore, shared_secret_digest + +_MAX_LINE_BYTES = 1 << 20 +_KEEPALIVE_PROBES = 6 +_AUTHORITY_WAIT_WARNING_S = 30.0 + + +class Lines: + """Line framing for one side of a liveness connection.""" + + def __init__(self) -> None: + self._buffer = b"" + + @staticmethod + def encode(payload: bytes) -> bytes: + return payload + b"\n" + + def decode(self, data: bytes) -> list[bytes]: + """Return the payloads of the lines that ``data`` completes.""" + *lines, self._buffer = (self._buffer + data).split(b"\n") + if len(self._buffer) > _MAX_LINE_BYTES: + raise ValueError("a liveness line is too long") + return lines + + +@dataclass +class LivenessConnection: + """A process's liveness connection to the authority, closed at the end of a ``with`` block.""" + + sock: socket.socket + lines: Lines + + def __enter__(self) -> "LivenessConnection": + return self + + def __exit__(self, *exc_info) -> None: + self.sock.close() + + +def configure_connection(sock: socket.socket) -> None: + """Set up a liveness connection's socket. + + It sends each record at once and fails once its peer is silent for ``DEAD_TIMEOUT_S``. + """ + # Both ends send small records, which Nagle's algorithm would hold until the peer's delayed + # acknowledgment. + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + idle_s = int(DEAD_TIMEOUT_S / 2) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle_s) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, idle_s // _KEEPALIVE_PROBES) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, _KEEPALIVE_PROBES) + # Keepalive probes start only on an idle connection, so this bounds a send the peer never + # acknowledges. + if hasattr(socket, "TCP_USER_TIMEOUT"): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(DEAD_TIMEOUT_S * 1000)) + + +@functools.lru_cache(maxsize=1) +def _closed_in_forked_child() -> "weakref.WeakSet[socket.socket | IO[bytes]]": + resources = weakref.WeakSet() + + def close_all() -> None: + for resource in list(resources): + resource.close() + + os.register_at_fork(after_in_child=close_all) + return resources + + +def close_in_forked_child(resource: socket.socket | IO[bytes]) -> None: + """Close ``resource`` in every child that this process forks, without keeping it alive.""" + _closed_in_forked_child().add(resource) + + +def _warn_while_no_authority(store: SignedStore) -> None: + """Log once if the authority has not published its liveness address within the warning time.""" + from torch.distributed import DistStoreError + + try: + store.wait([LIVENESS_KEY], timedelta(seconds=_AUTHORITY_WAIT_WARNING_S)) + except DistStoreError: + logger.warning( + f"No membership authority has published its liveness address in the rendezvous " + f"store at {store.host}:{store.port} after " + f"{_AUTHORITY_WAIT_WARNING_S:.0f} s. Still waiting" + ) + + +def hello_token() -> str: + """Return the token by which the authority admits a process of this deployment.""" + return shared_secret_digest(b"liveness hello") + + +def _open_connection(store: SignedStore, announcement: str, timeout_s: float) -> LivenessConnection: + """Open a connection to the authority and send the hello that carries ``announcement``.""" + _warn_while_no_authority(store) + address = rendezvous_host_port(store.get(LIVENESS_KEY).decode()) + sock = socket.create_connection(address, timeout=timeout_s) + configure_connection(sock) + lines = Lines() + sock.sendall(lines.encode(f"{hello_token()} {announcement}".encode())) + sock.settimeout(None) + return LivenessConnection(sock, lines) + + +def open_liveness_connection( + store: SignedStore, rank: int, replacement: bool = False +) -> LivenessConnection: + """Open this process's liveness connection to the authority for ``rank``. + + Args: + store: The rendezvous store, where the authority publishes its liveness address. + rank: The rank this process runs. + replacement: Whether this process replaces a failed process of ``rank``. + """ + connection = _open_connection(store, f"{rank} {int(replacement)}", HELLO_TIMEOUT_S) + close_in_forked_child(connection.sock) + return connection + + +def open_node_supervisor_connection( + store: SignedStore, + node: int, + ranks: list[int], + host: str, + timeout_s: float = HELLO_TIMEOUT_S, +) -> LivenessConnection: + """Open a node supervisor's connection for ``node``. Closing it fails ``ranks``. + + Args: + store: The rendezvous store, where the authority publishes its liveness address. + node: The node that the supervisor runs on. + ranks: The ranks that the supervisor starts on its node. + host: The node's host name, which the authority logs. + timeout_s: How long to wait for the connection to the authority. + """ + return _open_connection(store, f"node {node} {','.join(map(str, ranks))} {host}", timeout_s) + + +def open_store(address: str, timeout_s: float, secret: Optional[str] = None) -> SignedStore: + """Open a client for the store that the authority hosts at ``host:port``. + + The client signs and verifies values with ``secret`` (hex), by default ``SHARED_SECRET_ENV``. + """ + from torch.distributed import TCPStore + + host, port = rendezvous_host_port(address) + store = TCPStore( + host, + port, + is_master=False, + timeout=timedelta(seconds=timeout_s), + wait_for_workers=False, + ) + return SignedStore(store, secret) + + +def rendezvous_store(timeout_s: float = DEAD_TIMEOUT_S) -> SignedStore: + """Open a rendezvous store client from this process's environment. + + Raises: + ValueError: The environment holds no rendezvous address or shared secret. + """ + return open_store(rendezvous_address(), timeout_s) + + +class MembershipClient: + """A rank's thread that keeps its liveness connection, delivers records and sends reports. + + The thread ends once a record drops this process from the membership, and leaves the connection + open: the authority admits a replacement for the rank only once it closes, when the owner calls + :meth:`stop` or exits. + """ + + def __init__( + self, + rank: int, + store: SignedStore, + on_record: Callable[[MembershipRecord], None], + replacement: bool = False, + on_error: Optional[Callable[[Exception], None]] = None, + ) -> None: + """Prepare the client. :meth:`start` connects to the authority. + + Args: + rank: The rank this process runs. + store: The rendezvous store, where the authority publishes its liveness address. + on_record: Called on the client thread with each record the authority sends. + replacement: Whether this process replaces a failed process of ``rank``. + on_error: Called on the client thread with the error that ended it, if any. + """ + self.rank = rank + self._store = store + self._on_record = on_record + self._on_tick: Optional[Callable[[], None]] = None + self._on_error = on_error + self._replacement = replacement + self._report = RankReport() + self.record: Optional[MembershipRecord] = None + self.incarnation: Optional[int] = None + self._lock = threading.Lock() + self._delivery = threading.Lock() + self._stop = threading.Event() + self._wake_read, self._wake_write = os.pipe() + os.set_blocking(self._wake_read, False) + os.set_blocking(self._wake_write, False) + self._thread = threading.Thread( + target=self._run, name=f"MembershipClient{rank}", daemon=True + ) + self._sock: Optional[socket.socket] = None + self._lines: Optional[Lines] = None + self._received: list[bytes] = [] + + def start(self) -> None: + """Connect, take the authority's first record and start the thread. + + The first record sets ``record`` and ``incarnation`` at once. The thread also delivers it + to ``on_record``. + """ + connection = open_liveness_connection(self._store, self.rank, self._replacement) + self._sock, self._lines = connection.sock, connection.lines + try: + self._sock.settimeout(HELLO_TIMEOUT_S) + while not self._received: + data = self._sock.recv(1 << 16) + if not data: + raise ConnectionError( + f"the membership authority refused this process of rank {self.rank}" + ) + self._received = self._lines.decode(data) + self._sock.settimeout(None) + self.record = MembershipRecord.model_validate_json(self._received[0]) + self.incarnation = self.record.incarnations[self.rank] + self._send_report() + except BaseException: + # The authority admits a replacement for this rank only once the connection closes. + self._sock.close() + self._sock = None + raise + self._thread.start() + + def stop(self) -> None: + self._stop.set() + self._wake() + if self._thread.ident is not None: + self._thread.join() + if self._sock is not None: + self._sock.close() + # A report() racing this stop must not write to a closed or reused descriptor. + with self._lock: + os.close(self._wake_read) + os.close(self._wake_write) + self._wake_write = None + + def bind( + self, + on_record: Callable[[MembershipRecord], None], + on_tick: Optional[Callable[[], None]] = None, + on_error: Optional[Callable[[Exception], None]] = None, + ) -> None: + """Replace the callbacks, delivering the newest record to the new ``on_record`` at once.""" + with self._delivery: + self._on_record, self._on_tick, self._on_error = on_record, on_tick, on_error + if self.record is not None: + on_record(self.record) + + def report(self, **changes) -> None: + """Update report fields and wake the thread to send it. + + Raises: + pydantic.ValidationError: ``changes`` names a field the report lacks or has a value of + the wrong type. + """ + with self._lock: + self._report = RankReport.model_validate({**self._report.model_dump(), **changes}) + self._wake() + + def _wake(self) -> None: + with self._lock: + if self._wake_write is None: + return + try: + os.write(self._wake_write, b"\0") + except BlockingIOError: + pass + + def _send_report(self) -> None: + with self._lock: + report = self._report + self._sock.sendall(self._lines.encode(message_bytes(report))) + + def _run(self) -> None: + try: + self._serve() + return + except (OSError, ValueError) as error: + logger.error(f"Membership client of rank {self.rank} lost the authority: {error!r}") + failure = error + except _CallbackError as error: + failure = error.__cause__ + logger.error( + f"The {error} callback of the membership client of rank {self.rank} failed:\n" + + "".join(traceback.format_exception(failure)) + ) + # The thread ends here, so any other error must reach the owner too, or the rank would + # keep a connection that no longer reports. + except Exception as error: + failure = error + logger.error( + f"Membership client of rank {self.rank} failed:\n" + + "".join(traceback.format_exception(failure)) + ) + self._sock.close() + if self._on_error is not None: + self._on_error(failure) + + def _serve(self) -> None: + listed = False + tick_s = heard_s = time.monotonic() + lines, self._received = self._received, [] + while not self._stop.is_set(): + # A heartbeat line is empty. + for line in filter(None, lines): + record = MembershipRecord.model_validate_json(line) + with self._delivery: + self.record = record + _call("record", self._on_record, record) + member = record.is_member(self.rank, self.incarnation) + if listed and not member: + return + listed = member + watched = [self._sock, self._wake_read] + readable, _, _ = select.select(watched, [], [], max(0.0, tick_s - time.monotonic())) + if ( + self._sock not in readable + and time.monotonic() - heard_s > AUTHORITY_SILENCE_TIMEOUT_S + ): + # This thread can wait long for a CPU or the GIL after its select returns, while the + # authority's lines arrive. Look again before taking the authority as silent. + readable, _, _ = select.select(watched, [], [], 0.0) + if self._sock not in readable: + raise TimeoutError( + "the membership authority sent nothing for " + f"{AUTHORITY_SILENCE_TIMEOUT_S:g} s" + ) + lines = [] + if self._sock in readable: + data = self._sock.recv(1 << 16) + if not data: + raise ConnectionError("the membership authority closed the liveness connection") + lines = self._lines.decode(data) + if lines: + heard_s = time.monotonic() + due = time.monotonic() >= tick_s + if due: + tick_s = time.monotonic() + REPORT_INTERVAL_S + if self._on_tick is not None: + _call("tick", self._on_tick) + if due or self._wake_read in readable: + try: + os.read(self._wake_read, 1 << 12) + except BlockingIOError: + pass + self._send_report() + + +class _CallbackError(Exception): + """Raised when a callback of :class:`MembershipClient` fails, naming the callback.""" + + +def _call(name: str, callback: Callable[..., None], *args) -> None: + try: + callback(*args) + except Exception as error: + raise _CallbackError(name) from error diff --git a/tensorrt_llm/_torch/distributed/membership/records.py b/tensorrt_llm/_torch/distributed/membership/records.py new file mode 100644 index 000000000000..ff8e4c39e398 --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/records.py @@ -0,0 +1,284 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Membership records, rank reports and rank masks for Wide-EP fault tolerance. + +Every rank sends a :class:`RankReport` to the membership authority every ``REPORT_INTERVAL_S``. The +authority publishes each decision as a :class:`MembershipRecord` to every rank. A rank mask is an int +whose bit ``r`` is set when it holds rank ``r``. +""" + +import enum +import os +from collections.abc import Iterable +from typing import Literal, Optional + +from pydantic import BaseModel, ConfigDict + +from tensorrt_llm.llmapi.utils import _set_affinity_all_threads + +# The active mask has room for as many ranks as the all-to-all kernels support. +MAX_RANKS = 256 + +# The membership's timing. A report interval is shorter than a heartbeat interval, which is shorter +# than the silence timeout, which is shorter than the hang timeout. The admission timeout is shorter +# than the hang timeout too. The all-to-all kernels must wait at least twice the hang timeout. + +# Long enough that a slow but healthy host step is not taken as a hang. A hang found during a +# membership change can take a second timeout to remove, and twice this stays within +# the default 300 s executor hang detector and all-to-all timeouts. +HANG_TIMEOUT_S = 60.0 +REPORT_INTERVAL_S = 0.5 +# The authority sends an empty line to a rank that it has sent nothing for this long. A rank then +# hears from it at least every heartbeat plus report interval, so it outlasts an authority stall of +# the silence timeout less both (28.5 s). +HEARTBEAT_INTERVAL_S = 1.0 +# The age at which a rank takes the authority as silent. It is half the hang timeout, as the +# authority takes a rank's reports as stale after half of its own hang timeout. +AUTHORITY_SILENCE_TIMEOUT_S = HANG_TIMEOUT_S / 2 +# A joining rank that makes no progress in its admission for this long is left out, so a stalled +# joiner pauses the survivors far less than a hang timeout. +ADMISSION_TIMEOUT_S = 20.0 +# Longer than a network stall that TCP recovers from. A peer that stays connected but stops making +# progress is bounded by the hang timeout instead. +DEAD_TIMEOUT_S = 60.0 +HELLO_TIMEOUT_S = 10.0 +RENDEZVOUS_TIMEOUT_S = 600.0 +# Node supervisors start with the job and connect within seconds of the authority, so one still +# missing after this is a launch mistake. +NODE_SUPERVISOR_TIMEOUT_S = 60.0 + + +def remove_cpu_affinity_constraints() -> None: + """Let this process run on every CPU unless ``TLLM_NUMA_AWARE_WORKER_AFFINITY`` opts out. + + As ``configure_cpu_affinity`` does for a worker, override the inherited binding when the variable + is unset or ``1``, and keep it for any other value. A process that uses no GPU has no NUMA node to + pick, so it runs on every CPU. Without this, a process inherits its parent's binding, which a + launcher can set to the one CPU that rank 0 starts on. + """ + if os.environ.get("TLLM_NUMA_AWARE_WORKER_AFFINITY") in (None, "1"): + _set_affinity_all_threads(list(range(os.cpu_count()))) + + +def all_ranks(nranks: int) -> int: + """Return the rank mask of ranks ``0`` to ``nranks - 1``.""" + return (1 << nranks) - 1 + + +def has_rank(mask: int, rank: int) -> bool: + """Return whether rank mask ``mask`` holds ``rank``.""" + return bool(mask >> rank & 1) + + +def rank_mask(ranks: Iterable[int], nranks: int) -> int: + """Return the rank mask of ``ranks``, which must be ranks ``0`` to ``nranks - 1``. + + Raises: + ValueError: A rank is outside ``[0, nranks)``. + """ + mask = 0 + for rank in ranks: + if not 0 <= rank < nranks: + raise ValueError(f"rank must be in [0, {nranks}), got {rank}") + mask |= 1 << rank + return mask + + +# The host collectives over MNNVL memory number their rounds with an epoch in [1, _EPOCHS]. Epoch 0 +# is never sent, because the buffers start zeroed. A round uses the buffer half that its epoch's +# parity selects, so the epoch wraps from the even _EPOCHS to 1 to keep the parity alternating. +_EPOCHS = 0xFFFFFFFE + + +def next_epoch(epoch: int) -> int: + """Return the host-exchange epoch after ``epoch``.""" + return epoch % _EPOCHS + 1 + + +def latest_epoch(epochs: list[int]) -> int: + """Return the latest of ``epochs``. They must lie within half the epoch range of each other. + + The epochs compare on the circle relative to ``epochs[0]``, so 1 counts as later than + ``_EPOCHS``. + """ + return max(epochs, key=lambda epoch: (epoch - epochs[0] + _EPOCHS // 2) % _EPOCHS) + + +class MembershipState(enum.IntEnum): + """The state of a membership record. + + STARTING until every rank started, then COMMITTED while the ranks serve. A change goes through + PROPOSE (every active rank pauses and replies READY) and INSTALL (every active rank applies the + change and replies INSTALLED) back to COMMITTED. ABORTED ends the deployment with an error. + SHUTDOWN is a normal exit, in which a closed connection is not a failure. + """ + + COMMITTED = 0 + PROPOSE = 1 + INSTALL = 2 + ABORTED = 3 + SHUTDOWN = 4 + STARTING = 5 + + +class Reply(enum.IntEnum): + """A rank's reply to the current PROPOSE (READY) or INSTALL (INSTALLED), or FAILED.""" + + NONE = 0 + READY = 1 + INSTALLED = 2 + FAILED = 3 + + +class _Message(BaseModel): + """A membership protocol message, sent as JSON.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ResumeState(_Message): + """Where every rank continues after a membership change.""" + + iteration: int = 0 + # The host-collective epoch that every rank resumes from. Communication backends derive their + # per-round counters from it, so all ranks agree after a change. + epoch: int = 0 + + +class MembershipRecord(_Message): + """A membership decision by the authority. + + Attributes: + generation: Membership change counter, incremented by each PROPOSE. + state: The change's state. + active_mask: The ranks in the membership. + resume: Where the ranks resume, set by the INSTALL of a change. + incarnations: Current process per rank. 0 is the original, n the n-th replacement. + joining_mask: The active ranks that are replacements being added by this change. + given_up_mask: The ranks that no replacement is started for, as their node supervisor gave + them up or left. + reason: Why a change left a rank out, or why the deployment aborted. + """ + + generation: int + state: MembershipState + active_mask: int + resume: ResumeState = ResumeState() + incarnations: tuple[int, ...] = () + joining_mask: int = 0 + given_up_mask: int = 0 + reason: str = "" + + def is_active(self, rank: int) -> bool: + """Return whether ``rank`` is in the membership, whichever process serves it.""" + return has_rank(self.active_mask, rank) + + def is_joining(self, rank: int) -> bool: + """Return whether ``rank`` is a replacement that this change adds.""" + return has_rank(self.joining_mask, rank) + + def is_member(self, rank: int, incarnation: int) -> bool: + """Return whether ``rank`` is active with ``incarnation`` as its current process.""" + current = self.incarnations[rank] if rank < len(self.incarnations) else 0 + return self.is_active(rank) and current == incarnation + + +class NodeView(_Message): + """What the authority sends a node supervisor.""" + + record: MembershipRecord + # The ranks whose replacement made no startup progress for the rendezvous timeout. + stalled_replacements: tuple[int, ...] = () + # Rank mask of the ranks whose current process has a liveness connection to the authority. + connected_mask: int = 0 + + +class NodeReport(_Message): + """What a node supervisor last reported to the authority. + + Attributes: + given_up: The node's ranks that the supervisor gave up. + paused_mask: Rank mask of the node's ranks whose hang detector is paused and whose process + is not stopped. + received_s: When the authority received the report, on its monotonic clock. + """ + + given_up: tuple[int, ...] = () + paused_mask: int = 0 + received_s: float = 0.0 + + +class RankReport(_Message): + """What one rank last reported to the authority. + + Attributes: + progress: The executor's iteration, or during the startup the number of startup + collectives the rank completed. A replacement also counts the steps of its autotuner + warmup. + waiting_for: Rank mask of peers this rank currently waits on in a collective. + reply_generation: The generation that ``reply`` answers. + reply: The rank's reply to that record. + paused_at: The iteration and epoch at which the rank paused for the current change. + incarnation: The rank's process, set by the authority from the connection's hello. + received_s: When the authority received the report, on its monotonic clock. Reports + that stop arriving tell the authority that the process stopped. + waited_flags: (peer, round flag) for each all-to-all round that the rank waits on, so a + peer that still delivers new rounds is not hung. + waited_epochs: (peer, epoch) for each host-collective round that the rank waits on. + install_steps: Completed steps in the current INSTALL. Each one counts as progress. + joining_ready: A replacement is ready to join. It counts as its READY to the change that + adds it. + started: The rank finished its startup. + shutdown: Rank 0 shuts the deployment down. + """ + + progress: int = 0 + waiting_for: int = 0 + reply_generation: int = 0 + reply: Reply = Reply.NONE + paused_at: ResumeState = ResumeState() + incarnation: int = 0 + received_s: float = 0.0 + waited_flags: tuple[tuple[int, int], ...] = () + waited_epochs: tuple[tuple[int, int], ...] = () + install_steps: int = 0 + joining_ready: bool = False + started: bool = False + shutdown: bool = False + + +class TransactionSummary(_Message): + """An ended membership transaction, with its milestones in Unix milliseconds.""" + + generation: int + outcome: Literal["COMMITTED", "ABORTED"] + removed_ranks: tuple[int, ...] + # Why each removed rank was removed: "disconnected" (its liveness connection closed) or "hung". + causes: dict[int, str] = {} + added_ranks: tuple[int, ...] + # When the first removed rank stalled or disconnected, or when the first replacement was ready. + detected_unix_ms: float + began_unix_ms: float + ready_unix_ms: Optional[float] = None + ended_unix_ms: float + # Ended transactions per kind (failover, admission or abort), this one included. + changes: dict[str, int] = {} + # Why the deployment aborted, for an aborted transaction. + reason: str = "" + + +def message_bytes(message: _Message) -> bytes: + """Return ``message`` as JSON bytes.""" + return message.model_dump_json().encode() diff --git a/tensorrt_llm/_torch/distributed/membership/rendezvous.py b/tensorrt_llm/_torch/distributed/membership/rendezvous.py new file mode 100644 index 000000000000..4c11deb07289 --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/rendezvous.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rendezvous store communicators for Wide-EP fault tolerance.""" + +import hashlib +import os +import pickle # nosec B403 +from collections.abc import Callable, Sequence +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Optional + +from tensorrt_llm._torch.modules.dwdp.rendezvous import StoreComm +from tensorrt_llm._utils import OMPI_COMM_TYPE_HOST +from tensorrt_llm._wide_ep_fault_tolerance import RENDEZVOUS_ENV + +from .records import MAX_RANKS, REPORT_INTERVAL_S, rank_mask + +if TYPE_CHECKING: + from mpi4py import MPI + from torch.distributed import Store + + +LIVENESS_KEY = "liveness" +RECORD_KEY = "record" +TRANSACTION_KEY = "last_transaction" +# The Unix time in milliseconds since which the active ranks have waited on a rank that makes no +# progress, or empty while no rank stalls them. +STALLED_SINCE_KEY = "stalled_since" + + +# Ranks per store counter in an arrival mask. The store's counters are signed 64-bit integers. +_RANKS_PER_COUNTER = 63 + + +def rendezvous_address() -> str: + """Return the rendezvous address ``host:port`` that ``RENDEZVOUS_ENV`` holds. + + Raises: + ValueError: ``RENDEZVOUS_ENV`` is not set. + """ + address = os.environ.get(RENDEZVOUS_ENV) + if not address: + raise ValueError(f"Wide-EP fault tolerance requires host:port in {RENDEZVOUS_ENV}") + return address + + +def rendezvous_host_port(address: str) -> tuple[str, int]: + """Split a rendezvous address ``host:port`` into its host and port.""" + host, port = address.rsplit(":", 1) + return host, int(port) + + +def wait_for_keys( + store: "Store", keys: list[str], check_s: float, check: Callable[[], None] +) -> list[bytes]: + """Wait until ``keys`` are set and return their values. + + Call ``check`` every ``check_s`` while waiting. ``check`` may raise to stop waiting. + """ + from torch.distributed import DistStoreError + + while True: + try: + store.wait(keys, timedelta(seconds=check_s)) + break + except DistStoreError: + check() + return store.multi_get(keys) + + +class StoreGroup(StoreComm): + """A subset of an mpi4py communicator of the global ranks ``ranks``, over the rendezvous store. + + Only the members in ``active`` take part, and the others' values are None. While a call waits, + ``on_wait`` gets a mask of active members whose values are missing, and may raise to stop + waiting. + """ + + def __init__( + self, + store: "Store", + name: str, + rank: int, + ranks: Sequence[int], + on_wait: Callable[[int], None] = lambda waiting: None, + active: Optional[Sequence[int]] = None, + gpus_per_node: int = 1, + ) -> None: + """Build the group's communicator over ``store``. + + Args: + store: The rendezvous store client that holds the group's values. + name: The key prefix of the group's calls, the same on every member. + rank: This process's global rank, one of ``ranks``. + ranks: The group's global ranks, in communicator rank order. + on_wait: Called while a call waits, with a mask of the active members whose values + are missing, and with 0 once the call has them. + active: The members that take part, by default every member. + gpus_per_node: The ranks per node, which ``Split_type`` groups by. + """ + self.membership = tuple(ranks) + super().__init__( + store, + name, + self.membership.index(rank), + len(self.membership), + timedelta(seconds=REPORT_INTERVAL_S), + ) + self.rank = rank + self.exchange_failed = False + self._on_wait = on_wait + self._active = tuple(r for r in self.membership if active is None or r in active) + self._gpus_per_node = gpus_per_node + self._subgroups = 0 + + def allgather(self, value: Any) -> list[Any]: + """Return every member's value by communicator rank, with None for an inactive member. + + ``exchange_failed`` tells afterwards whether the exchange itself raised. + """ + return self._gather_values(value) + + def _gather_values(self, value: Any) -> list[Any]: + from torch.distributed import DistError + + self.exchange_failed = False + try: + if self._active == (self.rank,): + self._on_wait(0) + value = pickle.loads(pickle.dumps(value)) # nosec B301 + return [value if r == self.rank else None for r in self.membership] + try: + return super().allgather(value) + except DistError as error: + raise RuntimeError( + f"Rank {self.rank} lost the rendezvous store, which the membership authority " + "hosts" + ) from error + except BaseException: + self.exchange_failed = True + raise + + def barrier(self) -> None: + self._gather_values(None) + + Barrier = barrier + + def bcast(self, obj: Any, root: int = 0) -> Any: + """Return ``obj`` from communicator rank ``root``, or None if that rank is not active.""" + return self._gather_values(obj if self.Get_rank() == root else None)[root] + + def allreduce(self, obj: Any, op: Optional["MPI.Op"] = None) -> Any: + """Reduce the active members' ``obj`` by ``MPI.SUM`` (default), ``MIN`` or ``MAX``.""" + from mpi4py import MPI + + op = MPI.SUM if op is None else op + for mpi_op, reduce in ((MPI.SUM, sum), (MPI.MIN, min), (MPI.MAX, max)): + if op == mpi_op: + return reduce(value for value in self._gather_values(obj) if value is not None) + raise NotImplementedError(f"StoreGroup reduces by MPI.SUM, MPI.MIN or MPI.MAX, not {op}") + + def Free(self) -> None: + """Do nothing. The group's exchanges share this process's store client.""" + + def subgroup(self, ranks: Sequence[int]) -> "StoreGroup": + """Return the subgroup of ``ranks``, which must include this rank.""" + ranks = tuple(ranks) + digest = hashlib.sha256(repr(ranks).encode()).hexdigest()[:16] + self._subgroups += 1 + return StoreGroup( + self._store, + f"{self._name}/{self._subgroups}-{digest}", + self.rank, + ranks, + self._on_wait, + self._active, + self._gpus_per_node, + ) + + def Split(self, color: int = 0, key: int = 0) -> "StoreGroup": + """Split as ``MPI.Comm.Split`` does. + + The subgroup leaves out the inactive members, whose colors are unknown. + """ + splits = self.allgather((color, key)) + ranks = sorted( + (split[1], index, rank) + for index, (rank, split) in enumerate(zip(self.membership, splits)) + if split is not None and split[0] == color + ) + return self.subgroup(rank for _, _, rank in ranks) + + def Split_type( + self, split_type: int, key: int = 0, info: Optional["MPI.Info"] = None + ) -> "StoreGroup": + """Return the members on this rank's node, supporting only the node split types. + + The subgroup orders its members by global rank; ``key`` and ``info`` are ignored. + """ + from mpi4py import MPI + + if split_type not in (MPI.COMM_TYPE_SHARED, OMPI_COMM_TYPE_HOST): + raise NotImplementedError( + f"StoreGroup splits by node only, not by split type {split_type}" + ) + node = self.rank // self._gpus_per_node + return self.subgroup(r for r in self.membership if r // self._gpus_per_node == node) + + def _gather(self, prefix: str) -> list[Optional[bytes]]: + """Return the active members' payloads for call ``prefix``, None for the others.""" + self._store.add(self._counter(prefix, self.rank), 1 << self.rank % _RANKS_PER_COUNTER) + index = {r: i for i, r in enumerate(self.membership)} + try: + values = self._wait(prefix, [f"{prefix}/{index[r]}" for r in self._active]) + finally: + self._on_wait(0) + self._delete_calls( + self._seq - 2, self._seq - 1, [self.Get_rank()], counters=self.rank == min(self._active) + ) + by_rank = dict(zip(self._active, values)) + return [by_rank.get(r) for r in self.membership] + + def _delete_calls(self, first: int, end: int, indices: list[int], counters: bool) -> None: + """Delete the values of the members ``indices`` in calls ``first`` to ``end`` (exclusive). + + Also delete the calls' arrival counters if ``counters``. + """ + for seq in range(max(0, first), end): + prefix = f"{self._name}/{seq}" + for index in indices: + self._store.delete_key(f"{prefix}/{index}") + if counters: + for key in self._counters(prefix): + self._store.delete_key(key) + + def _wait(self, prefix: str, keys: list[str]) -> list[bytes]: + active_mask = rank_mask(self._active, MAX_RANKS) + + def check() -> None: + arrived = sum( + self._store.add(key, 0) << int(key.rsplit("/", 1)[1]) * _RANKS_PER_COUNTER + for key in self._counters(prefix) + ) + self._on_wait(active_mask & ~arrived) + + return wait_for_keys(self._store, keys, REPORT_INTERVAL_S, check) + + def _counters(self, prefix: str) -> list[str]: + return sorted({self._counter(prefix, r) for r in self._active}) + + @staticmethod + def _counter(prefix: str, rank: int) -> str: + return f"{prefix}/arrived/{rank // _RANKS_PER_COUNTER}" diff --git a/tensorrt_llm/_torch/distributed/membership/signed_store.py b/tensorrt_llm/_torch/distributed/membership/signed_store.py new file mode 100644 index 000000000000..5241a5ecda9c --- /dev/null +++ b/tensorrt_llm/_torch/distributed/membership/signed_store.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Sign rendezvous store values with the Wide-EP deployment's shared secret.""" + +import hashlib +import hmac +import os +from datetime import timedelta +from typing import TYPE_CHECKING, Optional + +from tensorrt_llm._wide_ep_fault_tolerance import SHARED_SECRET_ENV +from tensorrt_llm.math_utils import ceil_div + +if TYPE_CHECKING: + from torch.distributed import Store + +_TAG_BYTES = hashlib.sha256().digest_size +_CHUNK_COUNT_BYTES = 4 +# Values above this size are split into chunks of this size, half the 8 MiB value limit of torch's +# default (libuv) TCPStore server. +_CHUNK_BYTES = 4 << 20 +SHARED_SECRET_BYTES = 32 + + +class UnsignedValueError(ValueError): + """A rendezvous store value is not signed with the deployment's shared secret.""" + + +def parse_shared_secret(secret: str) -> bytes: + """Parse the hex shared secret ``secret``. + + Raises: + ValueError: ``secret`` is not hex or is shorter than ``SHARED_SECRET_BYTES``. + """ + try: + parsed = bytes.fromhex(secret) + except ValueError: + parsed = b"" + if len(parsed) < SHARED_SECRET_BYTES: + raise ValueError( + f"Wide-EP fault tolerance requires a shared secret of at least {SHARED_SECRET_BYTES} " + f"bytes ({2 * SHARED_SECRET_BYTES} hex digits) in {SHARED_SECRET_ENV}" + ) + return parsed + + +def shared_secret_digest(label: bytes) -> str: + """Return the hex HMAC-SHA256 of ``label`` keyed by the shared secret in ``SHARED_SECRET_ENV``. + + Raises: + ValueError: ``SHARED_SECRET_ENV`` is not set or holds no valid shared secret. + """ + secret = parse_shared_secret(os.environ.get(SHARED_SECRET_ENV, "")) + return hmac.new(secret, label, hashlib.sha256).hexdigest() + + +class SignedStore: + """A rendezvous store client that signs the values it sets and verifies those it gets. + + A value's key holds an HMAC-SHA256 tag, keyed by a digest of the shared secret, over the key, a + chunk count and the value. A process without the secret cannot set a value that the deployment's + processes accept. Counters (``add``), deletes and waits carry no tag, and an older value that + the deployment set under a key verifies again if a process sets it back. A process that reaches + the store's port can therefore stall or end the deployment, or make it act on an older record, + as it can with an unsigned TCPStore: the store is meant for a trusted network. + + A value larger than ``_CHUNK_BYTES`` is set in chunks under ``/chunks/``, and ``key`` + holds their count and the value's SHA-256 digest in place of the value. The chunks are set + before ``key``, so a process that waits for ``key`` finds them. A value under one key has a + chunk count of 0. + """ + + def __init__(self, store: "Store", secret: Optional[str] = None) -> None: + """Sign the values of ``store`` with ``secret`` (hex), by default ``SHARED_SECRET_ENV``. + + Raises: + ValueError: ``secret`` is not a valid shared secret (:func:`parse_shared_secret`), or + is None and ``SHARED_SECRET_ENV`` is not set. + """ + secret = os.environ.get(SHARED_SECRET_ENV, "") if secret is None else secret + self._store = store + self._key = hmac.new( + parse_shared_secret(secret), b"rendezvous store values", hashlib.sha256 + ).digest() + + @property + def host(self) -> str: + return self._store.host + + @property + def port(self) -> int: + return self._store.port + + def set(self, key: str, value: bytes) -> None: + value = bytes(value) + chunks, head = 0, value + if len(value) > _CHUNK_BYTES: + chunks = ceil_div(len(value), _CHUNK_BYTES) + for i in range(chunks): + chunk = value[i * _CHUNK_BYTES : (i + 1) * _CHUNK_BYTES] + self._store.set(_chunk_key(key, i), chunk) + head = hashlib.sha256(value).digest() + signed = chunks.to_bytes(_CHUNK_COUNT_BYTES, "big") + head + self._store.set(key, self._tag(key, signed) + signed) + + def get(self, key: str) -> bytes: + """Wait until ``key`` is set and return its value. + + Raises: + UnsignedValueError: The value is not signed with the deployment's shared secret. + """ + return self._value(key, self._store.get(key)) + + def multi_get(self, keys: list[str]) -> list[bytes]: + """Return the values for ``keys``, as :meth:`get` does.""" + return [self._value(k, v) for k, v in zip(keys, self._store.multi_get(keys))] + + def check(self, keys: list[str]) -> bool: + return self._store.check(keys) + + def wait(self, keys: list[str], timeout: timedelta) -> None: + self._store.wait(keys, timeout) + + def add(self, key: str, amount: int) -> int: + return self._store.add(key, amount) + + def delete_key(self, key: str) -> bool: + """Delete ``key`` and its chunks, and return whether ``key`` was set.""" + deleted = self._store.delete_key(key) + chunk = 0 + while self._store.delete_key(_chunk_key(key, chunk)): + chunk += 1 + return deleted + + def _tag(self, key: str, value: bytes) -> bytes: + encoded = key.encode() + message = len(encoded).to_bytes(4, "big") + encoded + value + return hmac.new(self._key, message, hashlib.sha256).digest() + + def _value(self, key: str, stored: bytes) -> bytes: + """Return the value of ``key`` from its ``stored`` bytes and its chunks, if it has any.""" + tag, signed = stored[:_TAG_BYTES], stored[_TAG_BYTES:] + if not hmac.compare_digest(tag, self._tag(key, signed)): + raise UnsignedValueError( + f"Rendezvous store value {key!r} is not signed with this deployment's " + f"{SHARED_SECRET_ENV}, so another process set it" + ) + chunks = int.from_bytes(signed[:_CHUNK_COUNT_BYTES], "big") + head = signed[_CHUNK_COUNT_BYTES:] + if chunks == 0: + return head + value = b"".join(self._store.get(_chunk_key(key, i)) for i in range(chunks)) + if not hmac.compare_digest(hashlib.sha256(value).digest(), head): + raise UnsignedValueError( + f"The chunks of rendezvous store value {key!r} do not match its signed digest, so " + "another process set them or the value changed while it was read" + ) + return value + + +def _chunk_key(key: str, index: int) -> str: + return f"{key}/chunks/{index}" diff --git a/tensorrt_llm/_torch/modules/dwdp/rendezvous.py b/tensorrt_llm/_torch/modules/dwdp/rendezvous.py index 7aadd268300f..83c2c2d1a61b 100644 --- a/tensorrt_llm/_torch/modules/dwdp/rendezvous.py +++ b/tensorrt_llm/_torch/modules/dwdp/rendezvous.py @@ -102,7 +102,51 @@ def Get_size(self) -> int: ... -class StoreDwdpGroup: +class StoreComm: + """The ``allgather`` / ``Barrier`` subset of an mpi4py communicator, over a store. + + Call ``n`` sets this rank's pickled object under ``//`` and waits for the keys of + the other ranks, so every rank must issue the same sequence of collectives. + """ + + def __init__(self, store: Any, name: str, rank: int, size: int, timeout: timedelta) -> None: + self._store = store + self._name = name + self._rank = rank + self._size = size + self._timeout = timeout + self._seq = 0 + + def Get_rank(self) -> int: + return self._rank + + def Get_size(self) -> int: + return self._size + + def allgather(self, obj: Any) -> List[Any]: + """Gather one picklable object per rank, ordered by rank in the group.""" + prefix = f"{self._name}/{self._seq}" + self._seq += 1 + self._store.set(f"{prefix}/{self._rank}", pickle.dumps(obj)) + # Trust boundary: only the processes of this launch write the store, whose port must stay + # on the trusted cluster network, as for the MPI communicator this replaces. + return [ + None if payload is None else pickle.loads(payload) # nosec B301 + for payload in self._gather(prefix) + ] + + def Barrier(self) -> None: + """Match ``MPI_Barrier``; waiting for every key is the synchronization.""" + self.allgather(None) + + def _gather(self, prefix: str) -> List[Optional[bytes]]: + """Return every rank's payload of the call with key prefix ``prefix``, by rank.""" + keys = [f"{prefix}/{r}" for r in range(self._size)] + self._store.wait(keys, self._timeout) + return [self._store.get(key) for key in keys] + + +class StoreDwdpGroup(StoreComm): """The ``allgather`` / ``Barrier`` / ``Free`` subset of an mpi4py communicator, over a TCPStore. Global DWDP rank 0 hosts the store for every group; keys are namespaced by group id, so @@ -127,14 +171,11 @@ def __init__( if global_rank < 0: raise ValueError(f"global_rank must be non-negative, got {global_rank}") - self._size = dwdp_size self._global_rank = global_rank self._num_workers = num_groups * dwdp_size self._setup_done = False - self._rank = global_rank % dwdp_size self._group_id = global_rank // dwdp_size - self._seq = 0 - self._timeout = timedelta(seconds=timeout_s) + timeout = timedelta(seconds=timeout_s) if store is None: from torch.distributed import TCPStore @@ -144,10 +185,12 @@ def __init__( host_name=master_addr, port=master_port, is_master=global_rank == 0, - timeout=self._timeout, + timeout=timeout, wait_for_workers=False, ) - self._store = store + super().__init__( + store, f"dwdp/{self._group_id}", global_rank % dwdp_size, dwdp_size, timeout + ) # Two workers launched with the same TRTLLM_DWDP_RANK would overwrite each other's # payloads and silently pair the wrong expert slices; MPI rules this out by @@ -184,32 +227,11 @@ def from_env(cls, global_rank: int, dwdp_size: int, num_groups: int = 1) -> "Sto num_groups=num_groups, ) - def Get_rank(self) -> int: - return self._rank - - def Get_size(self) -> int: - return self._size - def allgather(self, obj: Any) -> List[Any]: """Gather one picklable object per rank, ordered by rank in the group.""" if self._store is None: raise RuntimeError("allgather on a freed DWDP group") - - # Keys are unique per (group, call); every rank must issue the same sequence of collectives. - prefix = f"dwdp/{self._group_id}/{self._seq}/" - self._seq += 1 - keys = [f"{prefix}{r}" for r in range(self._size)] - - self._store.set(keys[self._rank], pickle.dumps(obj)) - self._store.wait(keys, self._timeout) - # Trust boundary: the store is served by DWDP rank 0 and written only by the - # context workers of this launch; its port must stay on the trusted cluster - # network, as for the MPI communicator this replaces. - return [pickle.loads(self._store.get(key)) for key in keys] # nosec B301 - - def Barrier(self) -> None: - """Match ``MPI_Barrier``; waiting for every key is the synchronization.""" - self.allgather(None) + return super().allgather(obj) def mark_setup_done(self) -> None: """Record that this worker no longer needs the store for setup. Idempotent.""" diff --git a/tensorrt_llm/_wide_ep_fault_tolerance.py b/tensorrt_llm/_wide_ep_fault_tolerance.py index 55c2135372b5..52dd8218556c 100644 --- a/tensorrt_llm/_wide_ep_fault_tolerance.py +++ b/tensorrt_llm/_wide_ep_fault_tolerance.py @@ -14,5 +14,9 @@ # limitations under the License. """Definitions that the modules of Wide-EP fault tolerance share.""" +# The rendezvous store's host:port. The authority hosts the store. +RENDEZVOUS_ENV = "TLLM_FAULT_TOLERANCE_RENDEZVOUS" +# The hex secret that every deployment process holds. It derives the hello token. +SHARED_SECRET_ENV = "TLLM_FAULT_TOLERANCE_SHARED_SECRET" # The alternative that each error about an unmet fault tolerance requirement offers. TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" diff --git a/tests/unittest/_torch/distributed/store_group_test_utils.py b/tests/unittest/_torch/distributed/store_group_test_utils.py new file mode 100644 index 000000000000..cc62b2909f8a --- /dev/null +++ b/tests/unittest/_torch/distributed/store_group_test_utils.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Helpers of the tests of the membership and the collectives over the rendezvous store.""" + +import threading + +from torch.distributed import DistStoreError + + +class InMemoryStore: + """In-memory store with the TCPStore calls of the membership, with signed 64-bit counters.""" + + def __init__(self): + self.values = {} + self._lock = threading.Condition() + + def set(self, key, value): + with self._lock: + self.values[key] = bytes(value) + self._lock.notify_all() + + def wait(self, keys, timeout): + with self._lock: + if not self._lock.wait_for( + lambda: all(key in self.values for key in keys), timeout.total_seconds() + ): + raise DistStoreError(f"wait timeout after {timeout}, keys: {keys}") + + def get(self, key): + return self.values[key] + + def check(self, keys): + with self._lock: + return all(key in self.values for key in keys) + + def multi_get(self, keys): + with self._lock: + return [self.values[key] for key in keys] + + def add(self, key, amount): + with self._lock: + value = int(self.values.get(key, b"0")) + amount + assert -(1 << 63) <= value < 1 << 63, f"counter {key} overflows" + self.values[key] = str(value).encode() + self._lock.notify_all() + return value + + def delete_key(self, key): + with self._lock: + return self.values.pop(key, None) is not None + + +class Unpickled: + """A value whose unpickling runs code, as a forged pickle in the store would.""" + + calls = [] + + @staticmethod + def record(): + Unpickled.calls.append("unpickled") + + def __reduce__(self): + return (Unpickled.record, ()) diff --git a/tests/unittest/_torch/distributed/test_membership.py b/tests/unittest/_torch/distributed/test_membership.py new file mode 100644 index 000000000000..ddb8d24669d3 --- /dev/null +++ b/tests/unittest/_torch/distributed/test_membership.py @@ -0,0 +1,1756 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Wide-EP fault-tolerance authority's decisions, driven by simulated ranks. + +The authority process and the membership clients run over loopback sockets and a real TCPStore. +""" + +import contextlib +import logging +import os +import random +import select +import signal +import socket +import subprocess +import sys +import threading +import time +import weakref +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from _torch.distributed.store_group_test_utils import InMemoryStore +from pydantic import ValidationError +from torch.distributed import TCPStore + +from tensorrt_llm._torch.distributed import membership +from tensorrt_llm._torch.distributed.membership import ( + LIVENESS_KEY, + RECORD_KEY, + RENDEZVOUS_ENV, + SHARED_SECRET_ENV, + STALLED_SINCE_KEY, + TRANSACTION_KEY, + Authority, + AuthorityProcess, + MembershipClient, + MembershipRecord, + MembershipState, + NodeReport, + NodeView, + RankReport, + Reply, + ResumeState, + SignedStore, + TransactionSummary, + UnsignedValueError, + authority, + authority_process, + liveness, + open_liveness_connection, + open_node_supervisor_connection, + open_store, + rendezvous_store, +) +from tensorrt_llm._torch.distributed.membership.liveness import Lines, hello_token +from tensorrt_llm.logger import logger + +pytestmark = pytest.mark.cpu_only + +WORLD = 4 +ALL = (1 << WORLD) - 1 +# How long a test waits for a condition or for a thread to end. +TIMEOUT_S = 60.0 +JOIN_TIMEOUT_S = 10.0 + + +def _without(*ranks): + """Return the mask of every rank except ``ranks``.""" + return ALL & ~sum(1 << rank for rank in ranks) + + +@pytest.fixture(autouse=True) +def _report_interval(monkeypatch): + """Shorten the report and decision interval for the tests, and set the shared secret.""" + for module in (authority, authority_process, liveness): + monkeypatch.setattr(module, "REPORT_INTERVAL_S", 0.05) + monkeypatch.setenv(SHARED_SECRET_ENV, "ab" * 32) + + +class Cluster: + """Simulate ranks that report their progress and reply to the authority's records.""" + + def __init__(self, world=WORLD, **timeouts): + self.authority = Authority(world, **timeouts) + started = {r: RankReport(started=True) for r in range(world)} + record = self.authority.tick(0.0, started, (1 << world) - 1) + assert record.state == MembershipState.COMMITTED + self.now = 0.0 + self.alive = set(range(world)) # processes that hold their connection + self.hung = set() # connected but making no progress + self.stopped = set() # stopped by SIGSTOP, still connected, but no longer reporting + # Hang detector paused, as while waiting for requests, which the node supervisor reports. + self.paused = set() + self.shutdown = set() # ranks whose report says that the deployment ends normally + self.waiting_for = {} # rank -> mask it waits for + self.waited_flags = {} # rank -> ((peer, round flag its kernels wait for), ...) + # rank -> ((peer, epoch of the host-exchange round it waits in), ...) + self.waited_epochs = {} + self.installing = set() # completing a step of an install at every report + self.unprepared = set() # replacements that have not reported they are ready to join + self.install_reply = {r: Reply.INSTALLED for r in range(world)} + self.reports = {r: RankReport() for r in range(world)} + self.paused_at = {r: ResumeState(iteration=10 + r, epoch=100 + r) for r in range(world)} + self.published = [] + self.published_at = [] + + def step(self, seconds=0.05): + self.now += seconds + record = self.authority.record + for r in self.alive: + if r in self.stopped: + continue + old = self.reports[r] + reply, generation = old.reply, old.reply_generation + if r not in self.hung and record.is_active(r): + if record.state == MembershipState.PROPOSE: + reply, generation = Reply.READY, record.generation + elif record.state == MembershipState.INSTALL: + reply, generation = self.install_reply[r], record.generation + busy = r not in self.hung and r not in self.paused + self.reports[r] = RankReport( + received_s=self.now, + progress=old.progress + (1 if busy else 0), + waiting_for=self.waiting_for.get(r, 0), + waited_flags=self.waited_flags.get(r, ()), + waited_epochs=self.waited_epochs.get(r, ()), + install_steps=old.install_steps + (1 if r in self.installing else 0), + reply_generation=generation, + reply=reply, + paused_at=self.paused_at[r], + joining_ready=r not in self.unprepared, + shutdown=r in self.shutdown, + ) + connected = sum(1 << r for r in self.alive) + node = NodeReport(paused_mask=sum(1 << r for r in self.paused), received_s=self.now) + new = self.authority.tick(self.now, self.reports, connected, [node]) + if new is not None: + self.published.append(new) + self.published_at.append(self.now) + return new + + def run(self, seconds): + for _ in range(round(seconds / 0.05)): + self.step() + return self.authority.record + + def run_until(self, state): + """Step until the authority's record is in ``state``.""" + for _ in range(100): + if self.authority.record.state == state: + break + self.step() + assert self.authority.record.state == state + + +def states(cluster): + return [r.state for r in cluster.published] + + +TRANSACTION = [MembershipState.PROPOSE, MembershipState.INSTALL, MembershipState.COMMITTED] + + +@pytest.mark.parametrize("rank", [-1, WORLD, 1 << 40]) +def test_rank_mask_rejects_ranks_outside_world(rank): + with pytest.raises(ValueError, match=f"got {rank}$"): + membership.rank_mask([0, rank], WORLD) + + +def test_rank_mask_counts_repeated_rank_once(): + assert membership.rank_mask([1, 1, 3], WORLD) == 0b1010 + + +def test_healthy_ranks_change_nothing(): + cluster = Cluster() + cluster.run(5.0) + assert cluster.published == [] + assert cluster.authority.record == MembershipRecord( + generation=0, state=MembershipState.COMMITTED, active_mask=ALL + ) + + +@pytest.mark.parametrize( + "epochs, resume_epoch", [({}, 104), ({0: 0xFFFFFFFE, 1: 1, 3: 1}, 2)], ids=["plain", "wrapped"] +) +def test_closed_connection_removes_the_rank(epochs, resume_epoch): + cluster = Cluster() + for r, epoch in epochs.items(): # ranks 1 and 3 went on past the wrap + cluster.paused_at[r] = ResumeState(iteration=10 + r, epoch=epoch) + cluster.alive.discard(2) + record = cluster.run(1.0) + assert states(cluster) == TRANSACTION + assert record.generation == 1 + assert record.active_mask == _without(2) + # Resume from the furthest survivor, with the next epoch. + assert record.resume.iteration == 13 + assert record.resume.epoch == resume_epoch + + +@pytest.mark.parametrize("waits_in", ["all-to-all", "host exchange"]) +def test_delivered_rounds_are_not_a_hang(waits_in): + """Keep a rank whose iteration stalls while every round its peers wait for still arrives.""" + cluster = Cluster(hang_timeout_s=1.0) + rounds = cluster.waited_flags if waits_in == "all-to-all" else cluster.waited_epochs + cluster.hung.add(3) # its iteration does not advance + cluster.waiting_for.update({r: 1 << 3 for r in range(3)}) + for round_ in range(1, 13): # a new round every 0.25 s for 3 s + rounds.update({r: ((3, round_),) for r in range(3)}) + cluster.run(0.25) + assert cluster.published == [] + record = cluster.run(1.5) # the same round for longer than the hang timeout + assert record.state == MembershipState.COMMITTED + assert not record.is_active(3) + + +# The hang rule. Only the end of a wait-for chain that does not move for the hang timeout is hung. + + +def _first_change(cluster, seconds): + """Return the first record the authority publishes within ``seconds``, or None.""" + for _ in range(round(seconds / 0.05)): + if cluster.step() is not None: + return cluster.published[-1] + return None + + +def _release(cluster): + """Resume the active ranks after the authority removed the hung ones.""" + cluster.hung.clear() + cluster.waiting_for.clear() + cluster.waited_flags.clear() + cluster.waited_epochs.clear() + + +@pytest.mark.parametrize("stopped", [False, True]) +def test_only_wait_chain_end_is_hung(stopped): + """Remove only rank 2, stopped or hung, when rank 3 waits on it and ranks 0 and 1 on both.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.run(0.5) + (cluster.stopped if stopped else cluster.hung).add(2) + cluster.hung.update({0, 1, 3}) + cluster.waiting_for.update({0: 0b1100, 1: 0b1100, 3: 1 << 2}) + cluster.waited_flags[3] = ((2, 7),) + hung_s = cluster.now + record = _first_change(cluster, 2.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == _without(2) + assert 1.0 < cluster.now - hung_s <= 1.2 + _release(cluster) + assert cluster.run(1.0).active_mask == _without(2) + assert states(cluster) == TRANSACTION + + +@pytest.mark.parametrize("stopped_in", ["all-to-all", "host exchange", "stale wait"]) +def test_stopped_process_is_hung_anywhere(stopped_in): + """Remove rank 2 one hang timeout after it stops while waiting on a peer, keeping the peer. + + The peer then waits on rank 2, except for a stale wait, where the peer works on. + """ + cluster = Cluster(hang_timeout_s=1.0) + peer = 3 + cluster.waiting_for[2] = 1 << peer + if stopped_in == "all-to-all": + cluster.waited_flags[2] = ((peer, 7),) + elif stopped_in == "host exchange": + cluster.waited_epochs[2] = ((peer, 40),) + cluster.run(0.5) + cluster.stopped.add(2) + waiting = (0, 1) if stopped_in == "stale wait" else (0, 1, 3) + cluster.hung.update(waiting) + cluster.waiting_for.update({r: 1 << 2 for r in waiting}) + stopped_s = cluster.now + record = _first_change(cluster, 2.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == _without(2) + assert 1.0 < cluster.now - stopped_s <= 1.2 + _release(cluster) + assert cluster.run(1.0).state == MembershipState.COMMITTED + assert MembershipState.ABORTED not in states(cluster) + + +@pytest.mark.parametrize("reports", ["stop", "go on"]) +@pytest.mark.parametrize("paused", [True, False]) +def test_paused_rank_is_not_hung(paused, reports): + """Keep rank 2 while its latest report says its hang detector is paused, and remove it if not. + + Ranks 0, 1 and 3 wait on it in a host exchange for three hang timeouts. It makes no progress, + and either stops reporting, as in a paused call that holds the GIL, or reports on. + """ + cluster = Cluster(hang_timeout_s=1.0) + cluster.hung.add(2) + if paused: + cluster.paused.add(2) + cluster.run(0.1) + if reports == "stop": + cluster.stopped.add(2) + waiting = (0, 1, 3) + cluster.hung.update(waiting) + cluster.waiting_for.update({r: 1 << 2 for r in waiting}) + cluster.waited_epochs.update({r: ((2, 40),) for r in waiting}) + record = _first_change(cluster, 3.0) + if paused: + assert record is None + else: + assert record.state == MembershipState.PROPOSE + assert record.active_mask == _without(2) + + +def test_late_rank_after_idle_not_hung(): + """Keep a rank late to the first collective after an idle period, since idle time is no stall.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.paused.update(range(WORLD)) # rank 0 waits for requests and the others wait on it + cluster.waiting_for.update({r: 1 for r in range(1, WORLD)}) + cluster.run(5.0) + cluster.paused.clear() # a request arrives, and rank 2 is late to the next collective + cluster.hung.update(range(WORLD)) + cluster.waiting_for.update({r: 1 << 2 for r in (0, 1, 3)}) + cluster.run(0.3) + _release(cluster) + cluster.run(2.0) + assert cluster.published == [] + + +@pytest.mark.parametrize("how", ["hung", "stopped"]) +@pytest.mark.parametrize("phase", [MembershipState.PROPOSE, MembershipState.INSTALL]) +def test_hang_during_transaction_restarts_it(phase, how): + """Fail a rank that hangs in a phase like a crashed one, after the hang timeout.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.alive.discard(1) # a crash starts the transaction + cluster.run_until(phase) + getattr(cluster, how).add(3) + phase_s = cluster.now + record = _first_change(cluster, 2.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == 0b0101 + assert 1.0 < cluster.now - phase_s <= 1.2 + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == 0b0101 + assert MembershipState.ABORTED not in states(cluster) + + +@pytest.mark.parametrize("cause", ["install", "rank 0"]) +def test_failed_install_or_rank0_loss_aborts(cause): + """Abort on a failed survivor install or the loss of rank 0.""" + cluster = Cluster() + if cause == "install": + cluster.install_reply[1] = Reply.FAILED + cluster.alive.discard(0 if cause == "rank 0" else 3) + assert cluster.run(1.0).state == MembershipState.ABORTED + assert MembershipState.COMMITTED not in states(cluster) + assert cluster.authority.last_transaction.outcome == "ABORTED" + assert cluster.authority.last_transaction.changes == {"abort": 1} + assert cluster.authority.last_transaction.reason == cluster.authority.record.reason != "" + + +def test_hang_detection_dated_at_stall_start(): + """Date a hang's transaction from the start of the stall, which the authority publishes.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.hung.add(1) + cluster.waiting_for[0] = 1 << 1 + cluster.run(0.5) + assert cluster.authority.stalled_since_s == pytest.approx(0.05) + record = cluster.run(2.0) + assert record.state == MembershipState.COMMITTED + assert cluster.authority.stalled_since_s is None + summary = cluster.authority.last_transaction + assert summary.causes == {1: "hung"} + assert summary.began_unix_ms - summary.detected_unix_ms == pytest.approx(1050.0, abs=60.0) + + +def test_survivor_failure_during_admission_commits(): + """Commit when rank 1 fails while rank 2's replacement is admitted after ranks 2 and 3 failed.""" + cluster = Cluster() + cluster.alive -= {2, 3} + assert cluster.run(1.0).state == MembershipState.COMMITTED + _join(cluster, 2) # rank 3's replacement did not ask to join, so rank 2's does not wait for it + cluster.run_until(MembershipState.PROPOSE) + cluster.alive.discard(1) + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == 0b0101 + assert MembershipState.ABORTED not in states(cluster) + + +def test_commit_waits_for_every_install(): + """Never commit a generation that a rank replied READY to but did not install.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.alive.discard(3) + cluster.run(1.0) # in generation 1, every survivor's last reply is INSTALLED + cluster.alive.discard(2) + cluster.run_until(MembershipState.INSTALL) + cluster.hung.add(1) # replied READY to generation 2, never INSTALLED + record = cluster.run(3.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == 0b0001 + committed = [r.generation for r in cluster.published if r.state == MembershipState.COMMITTED] + assert committed == [1, 3] # never generation 2, which rank 1 did not install + + +def test_hang_clock_restarts_at_commit(): + """Give a rank waited on during the install a full hang timeout after the commit.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.alive.discard(3) + cluster.run_until(MembershipState.INSTALL) + cluster.hung.add(1) + cluster.waiting_for[2] = 1 << 1 + assert cluster.run(0.8).state == MembershipState.INSTALL + # It replies without progress (the hang detector stays paused through the change), and the + # change commits. + cluster.hung.discard(1) + cluster.paused.add(1) + assert _first_change(cluster, 0.2).state == MembershipState.COMMITTED + cluster.paused.discard(1) + cluster.hung.add(1) # its first step after the commit + cluster.run(0.5) + _release(cluster) + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == 0b0111 + assert states(cluster) == TRANSACTION + + +def test_progressing_install_is_not_hung(): + """Count each completed install step as progress, so only a stalled install is a hang.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.alive.discard(1) + cluster.run_until(MembershipState.INSTALL) + cluster.hung.add(3) # replies nothing and its iteration does not advance ... + cluster.installing.add(3) # ... but it completes install steps + cluster.run(3.0) + assert cluster.authority.record.state == MembershipState.INSTALL + cluster.installing.discard(3) + stalled_s = cluster.now + record = _first_change(cluster, 2.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == 0b0101 + assert 1.0 < cluster.now - stalled_s <= 1.2 + + +def test_stalled_joiner_left_out_early(): + """Leave out a joining rank that stalls in its admission after the admission timeout.""" + admission_s = 2.0 + cluster = Cluster(admission_timeout_s=admission_s) + cluster.alive.discard(2) + cluster.run(1.0) + _join(cluster, 2) + cluster.hung.add(2) # ready to join, but it never installs the change that adds it + cluster.run_until(MembershipState.INSTALL) + stalled_s = cluster.now + record = _first_change(cluster, admission_s + 1.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == _without(2) + assert cluster.now - stalled_s <= admission_s + 0.2 + + +def test_frozen_survivor_removed_before_joiner(): + """Remove a survivor that froze during an admission, not the joiner it holds up.""" + cluster = Cluster(hang_timeout_s=4.0, admission_timeout_s=1.0) + cluster.alive.discard(3) + cluster.run(1.0) + _join(cluster, 3) + cluster.run_until(MembershipState.INSTALL) + # Rank 2 freezes while it waits for the joiner's slot, and the joiner waits on rank 2's. + cluster.waiting_for.update({2: 1 << 3, 3: 1 << 2}) + cluster.hung.update({2, 3}) + cluster.step() + cluster.stopped.add(2) + record = _first_change(cluster, 8.0) + assert record.state == MembershipState.PROPOSE + assert record.active_mask == _without(2) + + +def test_unconnected_replacement_join_request_dropped(): + """Drop the join request of a replacement that never connects.""" + cluster = Cluster() + cluster.alive.discard(3) + cluster.run(1.0) + cluster.reports[3] = RankReport() + cluster.authority.request_join(3) # the replacement never connects + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + assert not record.is_active(3) + + +def _join(cluster, rank): + cluster.alive.add(rank) # a new process for the rank, with fresh reports + cluster.reports[rank] = RankReport() + cluster.authority.request_join(rank) + + +def test_stalled_replacement_startup_reported(): + """Report a replacement whose startup stalls for the rendezvous timeout, but not a slow one. + + The wait for the memory of the process it replaces is not a stall. + """ + cluster = Cluster(rendezvous_timeout_s=1.0) + cluster.hung.add(1) # Rank 1 holds the change that removes rank 3 before its commit. + cluster.alive.discard(3) + cluster.step() + _join(cluster, 3) + cluster.hung.add(3) + cluster.unprepared.add(3) + cluster.run(2.0) + assert cluster.authority.stalled_replacements == () + cluster.hung.discard(1) + cluster.alive -= {2, 3} + cluster.run(1.0) + assert not cluster.authority.record.active_mask & 0b1100 + _join(cluster, 2) + _join(cluster, 3) + cluster.unprepared.add(2) + cluster.run(3.0) + assert cluster.authority.stalled_replacements == (3,) + + +@pytest.mark.parametrize("late", ["not ready", "no report"]) +def test_replacements_added_in_one_transaction(late): + """Add a node's replacements together, waiting for one that is not ready or has not reported. + + The authority's clock is far from zero, as a monotonic clock is. + """ + cluster = Cluster() + cluster.now = 1e6 + cluster.alive -= {2, 3} + cluster.run(1.0) + cluster.published.clear() + _join(cluster, 2) + _join(cluster, 3) + if late == "not ready": + cluster.unprepared.add(3) + else: + del cluster.reports[3] # connected, but no report arrived yet + cluster.stopped.add(3) + cluster.run(2.0) + assert cluster.published == [] # waits for rank 3's replacement + cluster.unprepared.discard(3) + cluster.stopped.discard(3) + cluster.reports.setdefault(3, RankReport()) + record = cluster.run(1.0) + assert states(cluster) == TRANSACTION + assert record.active_mask == ALL + assert cluster.authority.last_transaction.added_ranks == (2, 3) + assert cluster.authority.last_transaction.changes == {"failover": 1, "admission": 1} + # The transaction is detected at rank 2's readiness and includes the wait for rank 3's. + summary = cluster.authority.last_transaction + assert summary.began_unix_ms - summary.detected_unix_ms >= 2000 + + +@pytest.mark.parametrize("failure", ["failed_reply", "hang"]) +@pytest.mark.parametrize("phase", [MembershipState.PROPOSE, MembershipState.INSTALL]) +def test_failed_replacement_left_out_of_change(phase, failure): + cluster = Cluster(hang_timeout_s=1.0) + cluster.alive.discard(2) + cluster.run(1.0) + _join(cluster, 2) + cluster.run_until(phase) + if failure == "failed_reply" and phase == MembershipState.INSTALL: + cluster.install_reply[2] = Reply.FAILED + else: + cluster.hung.add(2) # never replies again + if failure == "failed_reply": + cluster.reports[2] = cluster.reports[2].model_copy( + update=dict( + reply=Reply.FAILED, reply_generation=cluster.authority.record.generation + ) + ) + record = cluster.run(3.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == _without(2) + assert record.joining_mask == 0 + assert MembershipState.ABORTED not in states(cluster) + if failure == "failed_reply": + # The node supervisor reads why the change left the replacement out. + assert any("could not install" in r.reason for r in cluster.published) + + +@pytest.mark.parametrize("phase", [MembershipState.PROPOSE, MembershipState.INSTALL]) +@pytest.mark.parametrize("victim", [1, 2, 3]) +def test_second_failure_in_each_phase_commits(phase, victim): + cluster = Cluster() + first = 1 if victim != 1 else 2 + cluster.alive.discard(first) + cluster.run_until(phase) + cluster.alive.discard(victim) + record = cluster.run(3.0) + assert record.state == MembershipState.COMMITTED + assert not record.is_active(first) + assert not record.is_active(victim) + + +def test_rank_0_shutdown_report_ends_deployment(): + """End the deployment on rank 0's shutdown report only, so leaving ranks are not failures.""" + cluster = Cluster() + cluster.shutdown.add(1) + cluster.run(1.0) + assert cluster.published == [] + cluster.shutdown.add(0) + cluster.step() + assert cluster.authority.record.state == MembershipState.SHUTDOWN + cluster.alive.clear() + cluster.run(2.0) + assert [record.state for record in cluster.published] == [MembershipState.SHUTDOWN] + + +# The startup stays STARTING until every rank has started. Any loss before then aborts at once. + + +def _starting_reports(started=(), received_s=0.0): + return {r: RankReport(started=r in started, received_s=received_s) for r in range(WORLD)} + + +@pytest.mark.parametrize("failed_rank", [0, 2]) +def test_startup_rank_failure_aborts_at_once(failed_rank): + """Abort without waiting for the rendezvous timeout, and give the reason in the record.""" + authority = Authority(WORLD) + authority.tick(0.0, _starting_reports(), ALL) + record = authority.tick(0.05, _starting_reports(started={1}), _without(failed_rank)) + assert record.state == MembershipState.ABORTED + assert record.reason == f"ranks [{failed_rank}] failed during the startup" + + +@pytest.mark.parametrize("failure", ["absent", "stopped", "not started"]) +def test_startup_failure_aborts_at_rendezvous_timeout(failure): + """Abort when a rank fails the startup for the rendezvous timeout, and name it in the record.""" + authority = Authority(WORLD, rendezvous_timeout_s=60.0) + for second in range(0, 91): + connected = ALL + if failure == "absent": + reports = _starting_reports(started={0, 1, 2}, received_s=second) + connected = _without(3) + elif failure == "stopped": + reports = { + r: RankReport(received_s=second, progress=3, waiting_for=1 << 3) + for r in range(WORLD) + } + reports[3] = RankReport(received_s=min(second, 10), progress=3) # stopped at 10 s + else: + reports = { + r: RankReport(received_s=second, progress=3, started=r != 3) for r in range(WORLD) + } + record = authority.tick(float(second), reports, connected) + if second <= 60: + assert record is None + assert authority.record.state == MembershipState.ABORTED + if failure == "absent": + assert authority.record.reason == "ranks [3] did not reach the rendezvous within 60 s" + else: + assert authority.record.reason == "ranks [3] failed during the startup" + + +@pytest.mark.parametrize("peer", ["hangs", "progresses"]) +def test_startup_collective_hang_aborts(peer): + """Abort when a waited-on rank completes no startup collective for the rendezvous timeout.""" + authority = Authority(WORLD, rendezvous_timeout_s=600.0) + for second in range(0, 1300, 10): + reports = { + r: RankReport(received_s=second, progress=3, waiting_for=1 << 2) for r in range(WORLD) + } + reports[2] = RankReport( + received_s=second, progress=3 + (second if peer == "progresses" else 0) + ) + record = authority.tick(float(second), reports, ALL) + if second <= 600 or peer == "progresses": + assert record is None + if peer == "hangs": + assert authority.record.state == MembershipState.ABORTED + assert authority.record.reason == "ranks [2] failed during the startup" + + +def test_first_failover_cause_after_staggered_startup(): + """Give a hung rank's cause, not that it connected after the others during the startup.""" + cluster = Cluster(hang_timeout_s=1.0) + cluster.authority = Authority(WORLD, hang_timeout_s=1.0) + cluster.authority.tick(0.0, _starting_reports(), _without(1)) + record = cluster.authority.tick(0.05, _starting_reports(started=range(WORLD)), ALL) + assert record.state == MembershipState.COMMITTED + cluster.now = 0.05 + cluster.hung.add(1) + cluster.waiting_for[0] = 1 << 1 + assert cluster.run(2.0).state == MembershipState.COMMITTED + assert cluster.authority.last_transaction.removed_ranks == (1,) + assert cluster.authority.last_transaction.causes == {1: "hung"} + + +class SignedTcpStore: + """A real TCPStore that each thread reaches through its own signing client, as a process does.""" + + def __init__(self): + self._server = TCPStore("127.0.0.1", 0, is_master=True, wait_for_workers=False) + self._clients = threading.local() + self.host, self.port = "127.0.0.1", self._server.port + + def __getattr__(self, name): + if not hasattr(self._clients, "store"): + client = TCPStore(self.host, self.port, is_master=False, timeout=timedelta(seconds=30)) + self._clients.store = SignedStore(client) + return getattr(self._clients.store, name) + + +@contextlib.contextmanager +def _deployment(world, store=None, reply=None, **authority_options): + """Run an authority process and a client per rank, yield the store and clients, then shut down. + + The store is in memory unless ``store`` is given. The clients answer records with ``reply``, + by default as an executor does. + """ + store = InMemoryStore() if store is None else store + thread, returned = _serve( + store, AuthorityProcess(world, world, store, Authority(world, **authority_options)) + ) + clients = [] + try: + clients += [_member(store, r, world, reply=reply) for r in range(world)] + _until( + lambda: all( + client.record is not None and client.record.state == MembershipState.COMMITTED + for client in clients + ) + ) + yield store, clients + clients[0].report(shutdown=True) + _until( + lambda: ( + MembershipRecord.model_validate_json(store.get(RECORD_KEY)).state + == MembershipState.SHUTDOWN + ) + ) + # The store lives in the authority's process, which serves until every active rank has read + # the record. + thread.join(timeout=0.2) # A brief wait, after which it still serves. + assert thread.is_alive() + finally: + _leave(clients) + thread.join(JOIN_TIMEOUT_S) + assert [record.state for record in returned] == [MembershipState.SHUTDOWN] + + +def _serve(store, process): + """Run ``process`` on a thread until it serves, and return the thread and what its run returned. + + A run that raises a ``RuntimeError`` returns the error. + """ + returned = [] + + def run(): + try: + returned.append(process.run(host="127.0.0.1")) + except RuntimeError as error: + returned.append(error) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + _until(lambda: store.check([membership.LIVENESS_KEY])) + return thread, returned + + +def _until(condition, timeout_s=TIMEOUT_S): + deadline = time.monotonic() + timeout_s + while not condition(): + assert time.monotonic() < deadline, "timed out" + time.sleep(0.01) + + +def _member(store, rank, world, replacement=False, on_error=None, reply=None): + """Start a rank's client that replies to each record with ``reply``, or as an executor does.""" + reply = _reply if reply is None else reply + client = MembershipClient( + rank, + store, + lambda record: reply(client, record), + replacement=replacement, + on_error=on_error, + ) + client.start() + if replacement: + client.report(joining_ready=True) + else: + client.report(started=True) + return client + + +def _reply(client, record): + """Reply to ``record`` as an executor does.""" + if record.state == MembershipState.PROPOSE: + client.report(reply=Reply.READY, reply_generation=record.generation) + elif record.state == MembershipState.INSTALL: + client.report(reply=Reply.INSTALLED, reply_generation=record.generation) + + +def _refused(store, rank, replacement): + """Return whether the authority closes a liveness connection for ``rank``.""" + with open_liveness_connection(store, rank, replacement) as connection: + connection.sock.settimeout(10) + return connection.sock.recv(1 << 16) == b"" + + +def _leave(clients): + for client in clients: + if client._thread.is_alive(): + client.stop() + + +@pytest.mark.parametrize("end", [MembershipState.SHUTDOWN, MembershipState.ABORTED]) +def test_authority_ends_with_rank_connected(end): + """End the authority after rank 0 read SHUTDOWN, or a hang timeout after an abort.""" + store = InMemoryStore() + parent_read, parent_write = os.pipe() + process = AuthorityProcess(2, 2, store, Authority(2, hang_timeout_s=5.0), parent=parent_read) + thread, returned = _serve(store, process) + states_at_close = [] + clients = [] + try: + clients.append(_member(store, 0, 2)) + clients.append( + _member( + store, 1, 2, on_error=lambda error: states_at_close.append(clients[1].record.state) + ) + ) + _until(lambda: all(c.record is not None and _committed(c, 0b11) for c in clients)) + if end == MembershipState.SHUTDOWN: + clients[0].report(shutdown=True) + _until(lambda: clients[0].record.state == MembershipState.SHUTDOWN) + clients[0].stop() # The deployment aborts unless it shuts down. + if end == MembershipState.SHUTDOWN: + os.close(parent_write) # as when rank 0's process ends + _until(lambda: clients[1].record.state == end) + thread.join(TIMEOUT_S) + assert not thread.is_alive() + if end == MembershipState.SHUTDOWN: + _until(lambda: states_at_close) + assert states_at_close == [MembershipState.SHUTDOWN] + finally: + _leave(clients) + os.close(parent_read) + if end == MembershipState.ABORTED: + os.close(parent_write) + assert [record.state for record in returned] == [end] + + +def test_authority_aborts_only_at_end_of_rank_0_pipe(): + """Keep serving when bytes arrive on rank 0's pipe, and abort once the pipe ends.""" + store = InMemoryStore() + parent_read, parent_write = os.pipe() + process = AuthorityProcess(1, 1, store, Authority(1), parent=parent_read) + thread, returned = _serve(store, process) + clients = [] + try: + clients.append(_member(store, 0, 1)) + _until(lambda: _committed(clients[0], 0b1)) + os.write(parent_write, b"\n") + time.sleep(10 * authority.REPORT_INTERVAL_S) + assert _committed(clients[0], 0b1) + os.close(parent_write) + parent_write = None + _until(lambda: clients[0].record.state == MembershipState.ABORTED) + finally: + _leave(clients) + thread.join(TIMEOUT_S) + os.close(parent_read) + if parent_write is not None: + os.close(parent_write) + assert [record.state for record in returned] == [MembershipState.ABORTED] + assert returned[0].reason == "rank 0's process ended" + + +@pytest.mark.parametrize("end", [MembershipState.SHUTDOWN, MembershipState.ABORTED]) +def test_authority_process_runs_until_rank_0_ends(end): + """Run the authority process as rank 0 starts it, and end it as rank 0 does. + + The process writes the store's address to ``--address-fd`` once the store listens there, runs + on every CPU though started bound to one, ignores SIGINT and SIGTERM, and exits with 0 after a + shutdown, else with 1. + """ + # Reserved with SO_REUSEADDR until the process ends, so that the authority can still bind it. + reserved = socket.socket() + reserved.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + reserved.bind(("127.0.0.1", 0)) + address = f"127.0.0.1:{reserved.getsockname()[1]}" + env = { + **os.environ, + RENDEZVOUS_ENV: address, + "TRT_LLM_NO_LIB_INIT": "1", + "MPI4PY_RC_INITIALIZE": "false", + "CUDA_VISIBLE_DEVICES": "", + } + env.pop("TLLM_NUMA_AWARE_WORKER_AFFINITY", None) + allowed = os.sched_getaffinity(0) + read_fd, write_fd = os.pipe() + os.sched_setaffinity(0, {min(allowed)}) + try: + process = subprocess.Popen( + [ + sys.executable, + "-m", + "tensorrt_llm._torch.distributed.membership", + "--world-size=1", + "--gpus-per-node=1", + f"--address-fd={write_fd}", + ], + env=env, + pass_fds=(write_fd,), + stdin=subprocess.PIPE, + ) + finally: + os.sched_setaffinity(0, allowed) + os.close(write_fd) + clients = [] + try: + with os.fdopen(read_fd) as address_pipe: + assert select.select([address_pipe], [], [], TIMEOUT_S)[0] + assert address_pipe.readline() == f"{address}\n" + store = open_store(address, timeout_s=TIMEOUT_S) + store.wait([LIVENESS_KEY], timedelta(seconds=TIMEOUT_S)) + assert os.sched_getaffinity(process.pid) >= allowed + for signum in (signal.SIGINT, signal.SIGTERM): + process.send_signal(signum) + clients.append(_member(store, 0, 1)) + _until(lambda: _committed(clients[0], 0b1)) + if end == MembershipState.SHUTDOWN: + clients[0].report(shutdown=True) + else: + process.stdin.close() # as when rank 0's process ends + _until(lambda: clients[0].record.state == end) + clients[0].stop() + assert process.wait(TIMEOUT_S) == (0 if end == MembershipState.SHUTDOWN else 1) + finally: + _leave(clients) + if process.poll() is None: + process.kill() + process.wait() + process.stdin.close() + reserved.close() + + +def test_stall_published_in_unix_milliseconds(): + """Publish since when the ranks wait on a stalled rank in Unix milliseconds, then clear it.""" + with _deployment(2) as (store, clients): + began_ms = time.time() * 1e3 + clients[0].report(waiting_for=0b10) + _until(lambda: store.check([STALLED_SINCE_KEY]) and store.get(STALLED_SINCE_KEY)) + assert began_ms - 50 <= int(store.get(STALLED_SINCE_KEY)) <= time.time() * 1e3 + clients[0].report(waiting_for=0) + _until(lambda: store.get(STALLED_SINCE_KEY) == b"") + + +def test_node_report_keeps_paused_rank(): + """Keep a rank that others wait on while its node supervisor reports it paused, and no longer. + + Rank 1 makes no progress while rank 0 waits on it. Its node supervisor reports it paused every + report interval for three hang timeouts, and then reports it armed. + """ + with _deployment(2, hang_timeout_s=1.0) as (store, clients): + node = _NodeSupervisor(store, 0, [0, 1]) + node.until(lambda view: True) + clients[0].report(waiting_for=0b10) + kept_until_s = time.monotonic() + 3.0 + while time.monotonic() < kept_until_s: + node.report(paused_mask=0b10) + time.sleep(0.05) + assert _committed(clients[0], 0b11) + node.report() + _until(lambda: _committed(clients[0], 0b01)) + + +def test_rendezvous_store_signs_with_shared_secret(monkeypatch): + """Open the store named in the environment with a client that signs with the shared secret.""" + server = TCPStore("127.0.0.1", 0, is_master=True, wait_for_workers=False) + address = f"127.0.0.1:{server.port}" + monkeypatch.setenv(RENDEZVOUS_ENV, address) + rendezvous_store(timeout_s=TIMEOUT_S).set("key", b"value") + assert open_store(address, TIMEOUT_S).get("key") == b"value" + with pytest.raises(UnsignedValueError): + open_store(address, TIMEOUT_S, secret="cd" * 32).get("key") + + +def test_rendezvous_store_requires_address(monkeypatch): + """Refuse to open the rendezvous store without an address, naming the variable.""" + monkeypatch.delenv(RENDEZVOUS_ENV, raising=False) + with pytest.raises(ValueError, match=f"requires host:port in {RENDEZVOUS_ENV}"): + rendezvous_store(timeout_s=TIMEOUT_S) + + +def test_failed_client_start_closes_connection(monkeypatch): + """Close the liveness connection when the first record does not arrive, and stop() returns. + + The authority admits a replacement for the rank only once the connection closes. + """ + monkeypatch.setattr(liveness, "HELLO_TIMEOUT_S", 0.2) + store = InMemoryStore() + closed = threading.Event() + with socket.create_server(("127.0.0.1", 0)) as listener: + store.set(membership.LIVENESS_KEY, f"127.0.0.1:{listener.getsockname()[1]}".encode()) + + def silent_authority(): + connection, _ = listener.accept() + with connection: + while connection.recv(1 << 16): + pass + closed.set() + + thread = threading.Thread(target=silent_authority, daemon=True) + thread.start() + client = MembershipClient(0, store, lambda record: None) + with pytest.raises(TimeoutError): + client.start() + assert closed.wait(JOIN_TIMEOUT_S) + client.stop() + thread.join(JOIN_TIMEOUT_S) + + +@pytest.mark.parametrize("shutdown", [False, True], ids=["idle", "shutdown"]) +def test_authority_heartbeats_keep_ranks_connected(monkeypatch, shutdown): + """Keep the ranks of an idle or ending deployment connected with the authority's heartbeats.""" + monkeypatch.setattr(authority_process, "HEARTBEAT_INTERVAL_S", 0.1) + monkeypatch.setattr(liveness, "AUTHORITY_SILENCE_TIMEOUT_S", 2.0) + with _deployment(2) as (_, clients): + if shutdown: + clients[0].report(shutdown=True) + _until(lambda: all(c.record.state == MembershipState.SHUTDOWN for c in clients)) + time.sleep(2 * liveness.AUTHORITY_SILENCE_TIMEOUT_S) + # A client that took the authority as hung has ended. + assert all(client._thread.is_alive() for client in clients) + + +def test_authority_logs_past_blocked_output(monkeypatch): + """Log from the authority's loop without waiting for an output stream that blocks.""" + released, written = threading.Event(), [] + + class BlockedOutput(logging.Handler): + def emit(self, record): + released.wait(JOIN_TIMEOUT_S) + written.append(record.getMessage()) + + monkeypatch.setattr(logging.getLogger(logger.PREFIX), "handlers", [BlockedOutput()]) + listener = authority_process.log_from_own_thread() + try: + logger.error("Membership: logged while the output blocks") + assert written == [] + finally: + released.set() + listener.stop() + assert len(written) == 1 + + +def test_liveness_connections_configured(monkeypatch): + """Send records at once and enable TCP keepalive at both ends of the connections of a rank and + of a node supervisor.""" + configure, accepted = liveness.configure_connection, [] + monkeypatch.setattr( + authority_process, + "configure_connection", + lambda sock: configure(sock) or accepted.append(sock), + ) + with _deployment(1) as (store, _): + supervisor = _NodeSupervisor(store, 0, [0]) + supervisor.until(lambda view: True) + # The authority's ends of the rank's and the node supervisor's connections. + assert len(accepted) == 2 + for sock in (supervisor.connection.sock, *accepted): + assert sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + assert sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) + idle_s = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE) + interval_s = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL) + probes = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT) + assert idle_s + interval_s * probes == membership.DEAD_TIMEOUT_S + user_timeout_ms = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT) + assert user_timeout_ms == membership.DEAD_TIMEOUT_S * 1000 + + +@pytest.mark.parametrize("published", [False, True]) +def test_waiting_for_the_authority_is_logged(monkeypatch, published): + """Wait in the store and log once, naming the store, if no liveness address arrives in time.""" + monkeypatch.setattr(liveness, "_AUTHORITY_WAIT_WARNING_S", 0.05) + warnings = [] + monkeypatch.setattr(liveness.logger, "warning", warnings.append) + tcp_store = TCPStore("127.0.0.1", 0, is_master=True, wait_for_workers=False) + if published: + tcp_store.set(LIVENESS_KEY, b"") + store = Mock(host="127.0.0.1", port=tcp_store.port, wait=Mock(wraps=tcp_store.wait)) + liveness._warn_while_no_authority(store) + store.wait.assert_called_once() + assert len(warnings) == (0 if published else 1) + assert published or f"127.0.0.1:{tcp_store.port}" in warnings[0] + + +def test_unchanged_membership_sends_nothing(): + """Send no record and write nothing to the store while the membership does not change.""" + with _deployment(2) as (store, clients): + clients[1].stop() + _until(lambda: clients[0].record.active_mask == 0b01) + _until(lambda: clients[0].record.state == MembershipState.COMMITTED) + writes = Mock(wraps=store.set) + store.set = writes + # No transaction starts for a replacement that has not reported that it is ready to join. + with open_liveness_connection(store, 1, replacement=True) as connection: + received = b"" + deadline_s = time.monotonic() + 3 * membership.records.HEARTBEAT_INTERVAL_S + while (left_s := deadline_s - time.monotonic()) > 0: + connection.sock.settimeout(left_s) + with contextlib.suppress(socket.timeout): + data = connection.sock.recv(1 << 16) + assert data, "the authority closed the connection" + received += data + # Only the record that gives it its incarnation arrives, besides empty heartbeat lines. + assert len([line for line in connection.lines.decode(received) if line]) == 1 + writes.assert_not_called() # until the connection closes + + +def test_foreign_or_second_replacement_refused(monkeypatch): + """Refuse a hello with another deployment's token, or a rank's second connected replacement.""" + with _deployment(2) as (store, clients): + clients[1].stop() + _until(lambda: _committed(clients[0], 0b01)) + monkeypatch.setenv(SHARED_SECRET_ENV, "cd" * 32) + assert _refused(store, 1, True) + with pytest.raises(ConnectionError, match="refused this process of rank 1"): + _member(store, 1, 2, replacement=True) + monkeypatch.setenv(SHARED_SECRET_ENV, "ab" * 32) + with open_liveness_connection(store, 1, replacement=True) as earlier: + earlier.sock.settimeout(10) + assert earlier.sock.recv(1 << 16), "the replacement was refused" + assert _refused(store, 1, True) + assert _committed(clients[0], 0b01) + + +class _NodeSupervisor: + """Node supervisor connection that reads the views the authority sends.""" + + def __init__(self, store, node, ranks): + self.connection = open_node_supervisor_connection(store, node, ranks, f"host-{node}") + self.connection.sock.settimeout(10) + + def until(self, predicate) -> NodeView: + """Return the first view that satisfies ``predicate``.""" + while True: + data = self.connection.sock.recv(1 << 16) + assert data, "the authority closed the connection" + views = [ + NodeView.model_validate_json(line) for line in self.connection.lines.decode(data) + ] + if views and predicate(views[-1]): + return views[-1] + + def report(self, given_up=(), paused_mask=0): + """Report the ranks that the supervisor gave up and those it found paused.""" + line = NodeReport(given_up=given_up, paused_mask=paused_mask).model_dump_json().encode() + self.connection.sock.sendall(self.connection.lines.encode(line)) + + +@pytest.mark.parametrize("lost_by", ["close", "foreign given-up rank", "foreign paused rank"]) +def test_node_supervisor_loss_fails_its_ranks(monkeypatch, lost_by): + """Remove a lost supervisor's ranks at once, though their processes run, and mark them given up. + + The removal is one transaction. A record resent with only the given-up ranks changed starts none. + """ + error = Mock() + monkeypatch.setattr(authority.logger, "error", error) + with _deployment(4) as (store, clients): + states = [] + + def on_record(record): + states.append(record.state) + _reply(clients[0], record) + + clients[0].bind(on_record) + # Node 0's supervisor stays connected, because its loss would abort the deployment. + node_0 = _NodeSupervisor(store, 0, [0, 1]) + node_1 = _NodeSupervisor(store, 1, [2, 3]) + node_1.until(lambda view: True) + if lost_by == "close": + node_1.connection.sock.close() + elif lost_by == "foreign given-up rank": + node_1.report(given_up=(0,)) + else: + node_1.report(paused_mask=0b0001) + _until(lambda: _committed(clients[0], 0b0011)) + node_0.until(lambda view: view.record.active_mask == 0b0011) + assert clients[0].record.given_up_mask == 0b1100 + generation = clients[0].record.generation + transaction = store.get(TRANSACTION_KEY) + summary = TransactionSummary.model_validate_json(transaction) + assert (summary.generation, summary.removed_ranks) == (generation, (2, 3)) + assert summary.changes == {"failover": 1} + assert states.count(MembershipState.PROPOSE) == 1 + + # Node 1's supervisor connects again and gives up none of its ranks. + node_1 = _NodeSupervisor(store, 1, [2, 3]) + node_1.report() + _until(lambda: clients[0].record.given_up_mask == 0) + assert _committed(clients[0], 0b0011) + assert clients[0].record.generation == generation + assert states.count(MembershipState.PROPOSE) == 1 + assert store.get(TRANSACTION_KEY) == transaction + invalid = { + "foreign given-up rank": "node 1 gave up ranks [0], which are not all its own", + "foreign paused rank": ( + "node 1 reported the ranks of mask 0x1 paused, which are not all its own" + ), + } + if lost_by in invalid: + (message,) = [call.args[0] for call in error.call_args_list if "node 1" in call.args[0]] + assert message == ( + "Closing the connection of the supervisor of node 1, because it sent an invalid line: " + f"{invalid[lost_by]}. Its ranks [2, 3] count as failed" + ) + + +def test_lost_node_rejoins_in_one_change(): + """Admit a lost node's new supervisor and its replacements, which join in one change.""" + with _deployment(4) as (store, clients): + states = [] + + def on_record(record): + states.append(record.state) + _reply(clients[0], record) + + clients[0].bind(on_record) + node_0 = _NodeSupervisor(store, 0, [0, 1]) + node_1 = _NodeSupervisor(store, 1, [2, 3]) + node_1.until(lambda view: True) + node_1.connection.sock.close() + _until(lambda: _committed(clients[0], 0b0011)) + node_1 = _NodeSupervisor(store, 1, [2, 3]) + view = node_1.until(lambda view: True) + assert (view.record.state, view.record.active_mask) == (MembershipState.COMMITTED, 0b0011) + node_1.report() + _until(lambda: clients[0].record.given_up_mask == 0) + # The supervisor starts both replacements at once, so both connect before either is ready. + joiners = {} + for rank in (2, 3): + joiners[rank] = MembershipClient( + rank, + store, + lambda record, rank=rank: _reply(joiners[rank], record), + replacement=True, + ) + joiners[rank].start() + clients.append(joiners[rank]) + for joiner in joiners.values(): + joiner.report(joining_ready=True) + _until(lambda: _committed(clients[0], 0b1111)) + assert states.count(MembershipState.PROPOSE) == 2 # the failover and the admission + summary = TransactionSummary.model_validate_json(store.get(TRANSACTION_KEY)) + assert summary.added_ranks == (2, 3) + assert clients[0].record.incarnations == (0, 0, 1, 1) + node_0.until(lambda view: view.record.active_mask == 0b1111) + + +@pytest.mark.parametrize("node, ranks", [(1, [2, 3]), (2, [3])], ids=["same node", "same rank"]) +def test_second_node_supervisor_refused(node, ranks): + """Refuse a supervisor for a node, or a rank, that a connected supervisor holds.""" + with _deployment(4) as (store, _): + node_1 = _NodeSupervisor(store, 1, [2, 3]) + node_1.until(lambda view: True) + with open_node_supervisor_connection(store, node, ranks, f"host-{node}") as second: + second.sock.settimeout(10) + assert second.sock.recv(1 << 16) == b"", "a second supervisor was admitted" + # The authority still admits another node's supervisor. + node_0 = _NodeSupervisor(store, 0, [0, 1]) + assert node_0.until(lambda view: True).record.active_mask == 0b1111 + + +def test_given_up_ranks_published_during_change(monkeypatch): + """Publish the ranks a node supervisor gave up at once, or during a change with its next record.""" + proposed, resumed = threading.Event(), threading.Event() + decide = Authority.tick + + def tick(self, now_s, reports, connected_mask, node_reports=()): + if proposed.is_set() and not resumed.is_set(): + return None + record = decide(self, now_s, reports, connected_mask, node_reports) + if record is not None and record.state == MembershipState.PROPOSE: + proposed.set() + return record + + monkeypatch.setattr(Authority, "tick", tick) + with _deployment(4) as (store, clients): + node_0 = _NodeSupervisor(store, 0, [0, 1]) + node_1 = _NodeSupervisor(store, 1, [2, 3]) + node_1.until(lambda view: True) + received = [] + + def on_record(record): + received.append(record) + _reply(clients[0], record) + + clients[0].bind(on_record) + clients[3].stop() + _until(lambda: clients[0].record.state == MembershipState.PROPOSE) + node_1.report(given_up=(3,)) + node_0.until(lambda view: view.record.given_up_mask == 0b1000) + resumed.set() + _until(lambda: _committed(clients[0], 0b0111)) + assert [record.state for record in received].count(MembershipState.PROPOSE) == 1 + assert clients[0].record.given_up_mask == 0b1000 + node_1.report() + _until(lambda: clients[0].record.given_up_mask == 0) + stored = MembershipRecord.model_validate_json(store.get(RECORD_KEY)) + assert stored.given_up_mask == 0 + + +def test_node_view_names_connected_ranks(monkeypatch): + """Send node supervisors the ranks with a connected process, again once one leaves.""" + deciding = threading.Event() + deciding.set() + decide = Authority.tick + + def tick(self, now_s, reports, connected_mask, node_reports=()): + return ( + decide(self, now_s, reports, connected_mask, node_reports) + if deciding.is_set() + else None + ) + + monkeypatch.setattr(Authority, "tick", tick) + with _deployment(4) as (store, clients): + node_0 = _NodeSupervisor(store, 0, [0, 1]) + node_0.until(lambda view: view.connected_mask == 0b1111) + # No membership change follows the leave, so only the connected ranks change. + deciding.clear() + clients[3].stop() + view = node_0.until(lambda view: view.connected_mask == 0b0111) + assert view.record.active_mask == 0b1111 + deciding.set() + + +@pytest.mark.parametrize("connected_nodes", [[0], []], ids=["one", "none"]) +def test_startup_without_node_supervisor_aborts(monkeypatch, connected_nodes): + """Abort a startup in which a node's supervisor did not connect, naming the nodes and the fix.""" + monkeypatch.setattr(authority_process, "NODE_SUPERVISOR_TIMEOUT_S", 0.5) + store = InMemoryStore() + thread, returned = _serve(store, AuthorityProcess(4, 2, store, Authority(4))) + supervisors = [ + _NodeSupervisor(store, node, [2 * node, 2 * node + 1]) for node in connected_nodes + ] + try: + for supervisor in supervisors: + supervisor.until(lambda view: view.record.state == MembershipState.ABORTED) + thread.join(JOIN_TIMEOUT_S) + finally: + for supervisor in supervisors: + supervisor.connection.sock.close() + assert [record.state for record in returned] == [MembershipState.ABORTED] + missing = [node for node in (0, 1) if node not in connected_nodes] + assert returned[0].reason.startswith(f"the supervisors of nodes {missing} did not connect") + assert SHARED_SECRET_ENV in returned[0].reason + + +@pytest.mark.parametrize("store_type", [InMemoryStore, SignedTcpStore]) +def test_replacement_refused_until_rank_removed(monkeypatch, store_type): + """Refuse a replacement for rank 0, or for a rank until the membership removes it. + + The replacement gets the rank's next incarnation, and the first process is no longer admitted. + Also over a real store with signed values, as the deployment's processes use. + """ + deciding = threading.Event() + deciding.set() + connected = [] + decide = Authority.tick + + def tick(self, now_s, reports, connected_mask, node_reports=()): + connected.append(connected_mask) + return ( + decide(self, now_s, reports, connected_mask, node_reports) + if deciding.is_set() + else None + ) + + monkeypatch.setattr(Authority, "tick", tick) + with _deployment(3, store_type()) as (store, clients): + assert _refused(store, 0, True) + assert _refused(store, 2, True) + deciding.clear() + clients[2].stop() + _until(lambda: connected[-1] == 0b011) + assert clients[0].record.is_active(2) + assert _refused(store, 2, True) + deciding.set() + _until(lambda: _committed(clients[0], 0b011)) + clients.append(_member(store, 2, 3, replacement=True)) + assert clients[-1].incarnation == 1 + _until(lambda: _committed(clients[0], 0b111)) + assert clients[0].record.incarnations == (0, 0, 1) + assert _refused(store, 2, False) + + +def test_second_failure_during_failover_install(): + """Fail a second rank while a survivor installs the first one's failover, with real clients. + + Rank 3 holds its reply to the failover's INSTALL, so the INSTALL is open when rank 2 fails. + The failover's generation never commits, and the next change commits without both ranks. + """ + held, seen = [], [] + + def holding_reply(client, record): + if client.rank == 0: + seen.append((record.generation, record.state)) + if client.rank == 3 and record.state == MembershipState.INSTALL and not held: + held.append(record.generation) + return + _reply(client, record) + + with _deployment(4, reply=holding_reply) as (store, clients): + clients[1].stop() + _until(lambda: held) + clients[2].stop() + _until(lambda: _committed(clients[0], 0b1001)) + summary = TransactionSummary.model_validate_json(store.get(TRANSACTION_KEY)) + (failover,) = held + assert (failover, MembershipState.INSTALL) in seen + assert (failover, MembershipState.COMMITTED) not in seen + assert summary.generation > failover + assert summary.removed_ranks == (1, 2) + + +def test_authority_stamps_reports_on_receipt(monkeypatch): + """Check that the authority stamps each report with the time it received it.""" + stamps = [] + decide = Authority.tick + + def tick(self, now_s, reports, connected_mask, node_reports=()): + stamps.extend((now_s, report.received_s) for report in reports.values()) + return decide(self, now_s, reports, connected_mask, node_reports) + + monkeypatch.setattr(Authority, "tick", tick) + with _deployment(2): + _until(lambda: len(stamps) >= 4) + assert all(0.0 <= now_s - received_s < 10.0 for now_s, received_s in stamps) + + +def test_first_process_refused_after_startup(monkeypatch): + """Refuse a first process once the deployment started, also for a failed rank never replaced.""" + warning = Mock() + monkeypatch.setattr(authority.logger, "warning", warning) + with _deployment(3) as (store, clients): + clients[2]._sock.shutdown(socket.SHUT_RDWR) + _until(lambda: _committed(clients[0], 0b011)) + assert clients[0].record.incarnations == (0, 0, 0) + assert _refused(store, 2, False) + (message,) = [call.args[0] for call in warning.call_args_list if "liveness" in call.args[0]] + assert message == ( + "Closing a liveness connection: rank 2 joins only as a replacement once the deployment has " + "started" + ) + + +def test_replacement_outside_deployment_refused(monkeypatch): + """Refuse a replacement of a rank outside the deployment, which keeps serving.""" + warning = Mock() + monkeypatch.setattr(authority.logger, "warning", warning) + with _deployment(2) as (store, clients): + assert _refused(store, 2, True) + assert _committed(clients[0], 0b11) + (message,) = [call.args[0] for call in warning.call_args_list if "liveness" in call.args[0]] + assert message == "Closing a liveness connection: rank 2 is not in the deployment" + + +def _committed(client, active_mask): + """Return whether ``client``'s newest record is COMMITTED with ``active_mask``.""" + record = client.record + return record.state == MembershipState.COMMITTED and record.active_mask == active_mask + + +def test_authority_survives_misbehaving_peer(monkeypatch): + """Keep serving after a peer whose first record fails to send and who sends an invalid line.""" + with _deployment(2) as (store, clients): + clients[1].stop() + _until(lambda: _committed(clients[0], 0b01)) + host, port = store.get(membership.LIVENESS_KEY).decode().rsplit(":", 1) + hello = Lines.encode(f"{hello_token()} 1 1".encode()) + with socket.create_connection((host, int(port)), timeout=10) as peer: + send, address = socket.socket.send, peer.getsockname() + + def send_failing_to_peer(sock, data, *args): + if sock.getpeername() == address: + raise ConnectionResetError("the peer reset the connection") + return send(sock, data, *args) + + monkeypatch.setattr(socket.socket, "send", send_failing_to_peer) + peer.sendall(hello + b"not a report\n") + while peer.recv(1 << 16): + pass + clients.append(_member(store, 1, 2, replacement=True)) + _until(lambda: _committed(clients[0], 0b11)) + + +def test_authority_failure_aborts_with_reason(monkeypatch): + """Send the ranks an ABORTED record naming an unexpected authority error, then raise it.""" + store = InMemoryStore() + thread, raised = _serve(store, AuthorityProcess(2, 2, store, Authority(2))) + clients = [] + try: + clients += [_member(store, r, 2) for r in range(2)] + _until(lambda: clients[0].record is not None and _committed(clients[0], 0b11)) + node = _NodeSupervisor(store, 0, [0, 1]) + node.until(lambda view: True) + node.report(given_up=(1,)) + _until(lambda: clients[0].record.given_up_mask == 0b10) + monkeypatch.setattr(store, "set", Mock(side_effect=RuntimeError("the store failed"))) + clients[1].stop() + _until(lambda: clients[0].record.state == MembershipState.ABORTED) + thread.join(JOIN_TIMEOUT_S) + finally: + _leave(clients) + assert clients[0].record.reason == "the authority failed: RuntimeError('the store failed')" + assert clients[0].record.given_up_mask == 0b10 + assert [str(error) for error in raised] == ["the store failed"] + + +def test_fork_close_keeps_no_closed_socket(): + """Keep no reference to a closed liveness socket for the close in a forked child.""" + with _deployment(1) as (store, _): + with open_liveness_connection(store, 0) as connection: + sock = weakref.ref(connection.sock) + del connection + assert sock() is None + + +def _scripted_authority(listener, resend_s, stop): + """Answer a rank's hello with a record, then resend it every ``resend_s`` if set.""" + sock, _ = listener.accept() + with sock: + hello = b"" + while not hello.endswith(b"\n"): + hello += sock.recv(256) + lines = Lines() + record = MembershipRecord( + generation=0, state=MembershipState.COMMITTED, active_mask=0b1, incarnations=(0,) + ) + sock.sendall(lines.encode(record.model_dump_json().encode())) + while not stop.wait(resend_s or None): + sock.sendall(lines.encode(record.model_dump_json().encode())) + + +@contextlib.contextmanager +def _scripted_client(errors, resend_s=None): + """Yield a started client of rank 0, connected to a scripted authority. + + The authority closes the connection first on exit, so the client may then add an error. + """ + store = InMemoryStore() + stop = threading.Event() + with socket.create_server(("127.0.0.1", 0)) as listener: + store.set(membership.LIVENESS_KEY, f"127.0.0.1:{listener.getsockname()[1]}".encode()) + thread = threading.Thread( + target=_scripted_authority, args=(listener, resend_s, stop), daemon=True + ) + thread.start() + client = MembershipClient(0, store, lambda record: None, on_error=errors.append) + client.start() + try: + yield client + finally: + stop.set() + thread.join(JOIN_TIMEOUT_S) + _leave([client]) + + +@pytest.mark.parametrize("resend_s", [None, 0.1], ids=["silent", "busy"]) +def test_rank_ends_on_silent_authority(monkeypatch, resend_s): + """End a rank's client once the authority is silent for the silence timeout, not before.""" + silence_s = 2.0 + monkeypatch.setattr(liveness, "AUTHORITY_SILENCE_TIMEOUT_S", silence_s) + errors = [] + # Before the client's thread starts, which is when its silence timeout starts. + started_s = time.monotonic() + with _scripted_client(errors, resend_s): + if resend_s is None: + _until(lambda: errors) + assert time.monotonic() - started_s >= silence_s + assert isinstance(errors[0], TimeoutError) + else: + time.sleep(2 * silence_s) + assert not errors + + +@pytest.mark.parametrize( + "changes", [{"waiting_on": 0b10}, {"progress": "first"}], ids=["unknown", "mistyped"] +) +def test_report_rejects_invalid_field(changes): + """Refuse a report change that names no report field or has a value of the wrong type.""" + errors = [] + with _scripted_client(errors) as client: + with pytest.raises(ValidationError): + client.report(**changes) + assert not errors + + +def test_client_holds_first_record_after_start(monkeypatch): + """Hold the authority's first record once start() returns, before the thread delivers it.""" + monkeypatch.setattr(liveness.MembershipClient, "_serve", lambda self: self._stop.wait()) + with _scripted_client([]) as client: + assert client.record is not None + assert client.record.incarnations[0] == client.incarnation + + +def test_late_wakeup_keeps_live_authority(monkeypatch): + """Keep a client whose thread wakes after the silence timeout while the authority's lines wait.""" + silence_s = 1.0 + monkeypatch.setattr(liveness, "AUTHORITY_SILENCE_TIMEOUT_S", silence_s) + skipped_s = [0.0] + monkeypatch.setattr( + liveness, "time", SimpleNamespace(monotonic=lambda: time.monotonic() + skipped_s[0]) + ) + calls = [] + + def select_with_late_wakeup(watched, *rest): + calls.append(watched) + if len(calls) > 1: + return select.select(watched, *rest) + # The authority's line has arrived, but the thread wakes only after the silence timeout. + select.select(watched[:1], [], []) + skipped_s[0] += 2 * silence_s + return [], [], [] + + monkeypatch.setattr(liveness, "select", SimpleNamespace(select=select_with_late_wakeup)) + errors = [] + with _scripted_client(errors, resend_s=0.1): + _until(lambda: errors or len(calls) > 2) + assert not errors + + +def test_callback_error_logged_with_traceback(monkeypatch): + """Report an error of a client's callback with its traceback, not as a lost authority.""" + logged, errors = [], [] + monkeypatch.setattr(liveness.logger, "error", logged.append) + + def on_tick(): + raise AttributeError("a bug in the callback") + + with _deployment(2) as (_, clients): + clients[1].bind(lambda record: None, on_tick=on_tick, on_error=errors.append) + _until(lambda: errors) + assert isinstance(errors[0], AttributeError) + (message,) = [text for text in logged if "callback" in text] + assert message.startswith("The tick callback of the membership client of rank 1 failed") + assert "Traceback" in message + + +def test_client_thread_error_closes_connection(monkeypatch): + """Report any error that ends a client's thread, and close its liveness connection.""" + logged, errors = [], [] + monkeypatch.setattr(liveness.logger, "error", logged.append) + with _scripted_client(errors) as client: + monkeypatch.setattr(client._lines, "encode", Mock(side_effect=KeyError("a bug"))) + client.report(progress=1) + _until(lambda: errors) + assert isinstance(errors[0], KeyError) + assert client._sock.fileno() == -1 + (message,) = logged + assert message.startswith("Membership client of rank 0 failed") + assert "Traceback" in message + + +def test_transaction_summary_spans_propose_to_commit(): + cluster = Cluster() + cluster.run(1.0) + assert cluster.authority.last_transaction is None + cluster.alive.discard(2) + cluster.run_until(MembershipState.INSTALL) + cluster.alive.discard(3) # restarts the transaction, which keeps its start + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + began = cluster.published_at[0] + ended = cluster.published_at[-1] + summary = cluster.authority.last_transaction + assert (summary.generation, summary.outcome) == (record.generation, "COMMITTED") + assert (summary.removed_ranks, summary.added_ranks) == ((2, 3), ()) + assert summary.causes == {2: "disconnected", 3: "disconnected"} + assert summary.ended_unix_ms - summary.began_unix_ms == pytest.approx((ended - began) * 1e3) + assert ended - began > 0 + # A closed connection is proposed in the tick that sees it, and the simulated ranks install in + # one step. + assert summary.detected_unix_ms == summary.began_unix_ms + assert summary.ended_unix_ms - summary.ready_unix_ms == pytest.approx(50.0) + + +def _interleave(cluster, rng): + """Apply a random crash, hang, join, readiness or install outcome, and delay random reports.""" + record, r = cluster.authority.record, rng.randrange(1, cluster.authority.world_size) + event = rng.randrange(5) + if event == 0: + cluster.alive.discard(r) + elif event == 1 and r not in cluster.alive: + _join(cluster, r) + cluster.unprepared.add(r) + cluster.hung.discard(r) + cluster.install_reply[r] = Reply.INSTALLED + elif event == 2: + cluster.unprepared.discard(r) + elif event == 3 and rng.random() < 0.1: + cluster.hung.add(r) + elif event == 4 and (record.is_joining(r) or rng.random() < 0.01): + cluster.install_reply[r] = Reply.FAILED + cluster.stopped = {q for q in cluster.alive if rng.random() < 0.2} + + +@pytest.mark.parametrize("seed", range(10)) +def test_random_interleavings_keep_invariants(seed): + """Check the protocol's invariants after every tick of random failures, joins and late replies.""" + rng, world = random.Random(seed), 4 + seed % 5 + cluster = Cluster(world, hang_timeout_s=1.0) + expected = {MembershipState.PROPOSE: Reply.READY, MembershipState.INSTALL: Reply.INSTALLED} + record = cluster.authority.record + committed, joining = record.active_mask, 0 + for step in range(600): + if step < 400: + _interleave(cluster, rng) + else: + cluster.stopped, cluster.unprepared = set(), set() + previous, new = record, cluster.step() + record = cluster.authority.record + assert record.generation >= previous.generation + if record.state == MembershipState.PROPOSE and new is not None: + assert record.generation > previous.generation + if record.state == MembershipState.ABORTED: + # Only the loss of rank 0, or a surviving rank that fails its install, requires an abort. + survivors = [ + r for r in range(world) if previous.is_active(r) and not previous.is_joining(r) + ] + assert not record.is_active(0) or any( + cluster.install_reply[r] == Reply.FAILED for r in survivors + ), record.reason + return + assert record.active_mask & ~sum(1 << r for r in cluster.alive) == 0 + active = [r for r in range(world) if record.is_active(r)] + replied = [ + cluster.reports[r].reply_generation == record.generation + and cluster.reports[r].reply == expected.get(previous.state) + for r in active + ] + if record.state in expected: + joining = record.joining_mask + assert record.active_mask & ~committed & ~joining == 0 + assert new is not None or not all(replied) + elif new is not None: + assert previous.state == MembershipState.INSTALL + assert all(replied) + assert record.active_mask & ~committed & ~joining == 0 + committed = record.active_mask + assert record.state == MembershipState.COMMITTED diff --git a/tests/unittest/_torch/distributed/test_membership_hang_rule.py b/tests/unittest/_torch/distributed/test_membership_hang_rule.py new file mode 100644 index 000000000000..34979c628ed0 --- /dev/null +++ b/tests/unittest/_torch/distributed/test_membership_hang_rule.py @@ -0,0 +1,383 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The authority's hang rule over every small deployment's failures (no GPU, no I/O). + +A rank blocked in a collective waits on a peer, so only the end of a wait-for chain is hung. Each +case gives every rank one role and derives the waits that its reports carry: + +- waiting: running and blocked, waits on the blockers, or on the waiting ranks that wait on them; +- slow: running and making progress, waits on nobody; +- idle: running in a call with its hang detector paused, as rank 0 waiting for requests; +- crashed: its connection closes and it reports nothing more; +- hung: its main thread stops outside any wait, so its fresh reports wait on nobody; +- stopped: its reports stop, and the last one waits on nobody or on the ranks that now wait on it; +- paused: its reports stop, and its node supervisor reports its hang detector paused. + +A rank whose node supervisor reports its hang detector paused is never hung outside a change, also +once its reports stop, so an idle or paused rank that others wait on is kept. A change waits on a +paused rank, which never replies, as on any other failed rank. The blockers are any nonempty set of +non-waiting ranks. +Ranks 1 and above are interchangeable to the rule, so their roles, the blockers and the chained +ranks are enumerated up to swapping ranks of one role. The waited rounds (``waited_flags``, +``waited_epochs``) and the install steps stay empty, so a rank's progress is its iteration. +""" + +import itertools +import random + +import pytest + +from tensorrt_llm._torch.distributed.membership import ( + Authority, + MembershipState, + NodeReport, + RankReport, + Reply, + authority, +) + +pytestmark = pytest.mark.cpu_only + +HANG_TIMEOUT_S = 1.0 +STEP_S = HANG_TIMEOUT_S / 4 +# Every case settles within four hang timeouts. The cases run one hang timeout longer. +SETTLE_S = 4 * HANG_TIMEOUT_S +TICKS = round((SETTLE_S + HANG_TIMEOUT_S) / STEP_S) + +WAITING, SLOW, IDLE = "waiting", "slow", "idle" +CRASHED, HUNG, STOPPED, PAUSED = "crashed", "hung", "stopped", "paused" +SILENT = (STOPPED, PAUSED) +FAULTY = (CRASHED, HUNG, *SILENT) +RANK_ROLES = (WAITING, SLOW, IDLE, *FAULTY) + +_reports = {} + + +def _report(**fields): + """Return a cached report with ``fields``, since the cases send the same reports many times.""" + key = tuple(sorted(fields.items())) + if key not in _reports: + _reports[key] = RankReport(**fields) + return _reports[key] + + +def _mask(ranks): + return sum(1 << r for r in ranks) + + +def _groups(roles): + """Return the ranks grouped by role, rank 0 on its own, since the other ranks are alike.""" + groups = [[0]] + for r in range(1, len(roles)): + if r > 1 and roles[r] == roles[r - 1]: + groups[-1].append(r) + else: + groups.append([r]) + return groups + + +def _subsets(groups): + """Yield one mask per subset of the ranks in ``groups``, up to swapping alike ranks.""" + for sizes in itertools.product(*(range(len(group) + 1) for group in groups)): + yield _mask(r for group, size in zip(groups, sizes) for r in group[:size]) + + +def _cases(world): + """Yield ``(roles, waits, stale)`` for every role assignment and every wait they imply.""" + for rank0_role in RANK_ROLES: + for rest in itertools.combinations_with_replacement(RANK_ROLES, world - 1): + roles = (rank0_role,) + rest + groups = _groups(roles) + waiting = [g for g in groups if roles[g[0]] == WAITING] + others = [g for g in groups if roles[g[0]] != WAITING] + if not waiting: + yield from _variants(roles, {}) + continue + # Running ranks never wait on each other in a cycle, because collectives send before + # they wait. + for blockers in filter(None, _subsets(others)): + # The chained ranks wait on the other waiting ranks, not on the blockers. + for chained in _subsets(waiting): + first_level = _mask(r for g in waiting for r in g) & ~chained + if first_level: + waits = { + r: first_level if chained >> r & 1 else blockers + for g in waiting + for r in g + } + yield from _variants(roles, waits) + + +def _variants(roles, waits): + stopped = [r for r, role in enumerate(roles) if role == STOPPED] + yield roles, waits, {} + if stopped: + # A stopped rank's last report waits on the ranks that now wait on it. + yield roles, waits, {s: _mask(r for r in waits if waits[r] >> s & 1) for s in stopped} + + +def _run(world, roles, waits, stale): + """Drive the authority through the case and return ``(time, record)`` for each record.""" + auth = Authority(world, hang_timeout_s=HANG_TIMEOUT_S) + everyone = (1 << world) - 1 + started = {r: RankReport(started=True) for r in range(world)} + assert auth.tick(0.0, started, everyone).state == MembershipState.COMMITTED + connected = everyone & ~_mask(r for r in range(world) if roles[r] == CRASHED) + paused_mask = _mask(r for r, role in enumerate(roles) if role in (IDLE, PAUSED)) + published = [] + for tick in range(1, TICKS + 1): + record = auth.record + reports = {} + for r, role in enumerate(roles): + if role == CRASHED: + continue + if role in SILENT: + reports[r] = _report(received_s=STEP_S, waiting_for=stale.get(r, 0)) + elif role == HUNG: + reports[r] = _report(received_s=tick * STEP_S) + else: + reports[r] = _running_report(record, role, waits.get(r, 0), tick) + now_s = tick * STEP_S + node_report = NodeReport(paused_mask=paused_mask, received_s=now_s) + new = auth.tick(now_s, reports, connected, [node_report]) + if new is not None: + published.append((now_s, new)) + if new is not None and new.state == MembershipState.ABORTED: + break + # Once the running ranks commit without the failed ranks, they only make progress. + settled = record.generation and record.state == MembershipState.COMMITTED + if settled and now_s - published[-1][0] > HANG_TIMEOUT_S: + break + return published + + +def _running_report(record, role, waiting_for, tick): + """Return a running rank's report, which replies to each change and resumes after the commit.""" + if record.generation: + resumed = record.state == MembershipState.COMMITTED + reply = { + MembershipState.PROPOSE: Reply.READY, + MembershipState.INSTALL: Reply.INSTALLED, + }.get(record.state, Reply.NONE) + return _report( + received_s=tick * STEP_S, + progress=tick if resumed else 0, + reply=reply, + reply_generation=record.generation, + ) + if role == SLOW: + return _report(received_s=tick * STEP_S, progress=tick) + return _report(received_s=tick * STEP_S, waiting_for=waiting_for) + + +def _mismatch(world, roles, waits, stale, published): + """Return why ``published`` breaks the hang rule for the case, or None.""" + everyone = (1 << world) - 1 + running = _mask(r for r in range(world) if roles[r] not in FAULTY) + crashed = _mask(r for r in range(world) if roles[r] == CRASHED) + paused = _mask(r for r in range(world) if roles[r] == PAUSED) + waited_on = 0 + for mask in waits.values(): + waited_on |= mask + # The ends of the wait-for chains: the failed ranks that a waiting rank waits on, except the + # paused ones. + roots = waited_on & everyone & ~running & ~crashed & ~paused + if any(record.active_mask & running != running for _, record in published): + return "removed a running rank" + if not crashed and not roots: + return "changed membership without a failure" if published else None + if not published: + return "missed the failure" + first_s, first = published[0] + removed = everyone & ~first.active_mask + expected = crashed or roots + if removed != expected: + return f"first removed {removed:#b}, expected {expected:#b}" + # The failures start with the first reports, one step in. + if crashed: + in_time = first_s == STEP_S + else: + in_time = HANG_TIMEOUT_S < first_s - STEP_S <= HANG_TIMEOUT_S + 2 * STEP_S + if not in_time: + return f"first change at {first_s} s" + last_s, last = published[-1] + if last_s > SETTLE_S: + return f"still changing at {last_s} s" + if running & 1: + if last.state != MembershipState.COMMITTED or last.active_mask != running: + return f"settled on {last}" + elif last.state != MembershipState.ABORTED: + return f"did not abort without rank 0: {last}" + return None + + +@pytest.mark.parametrize("world", [2, 3, 4, 5]) +def test_only_chain_ends_are_hung(world): + """Remove exactly the crashed ranks or the ends of the wait-for chains, then every failed rank. + + A crash starts a change at once. Without one, the authority removes the hung or stopped ranks + that waiting ranks wait on, one hang timeout after they stop. The change then waits on every + other failed rank, which never replies, so the deployment settles on the running ranks, or + aborts without rank 0. A failed rank that no running rank waits on is slow, not hung. An idle + or paused rank is never hung outside a change, so it is kept while other ranks wait on it. + """ + mismatches = [] + count = 0 + for roles, waits, stale in _cases(world): + count += 1 + problem = _mismatch(world, roles, waits, stale, _run(world, roles, waits, stale)) + if problem is not None: + mismatches.append(f"roles {roles}, waits {waits}, stale {stale}: {problem}") + assert count > 0 + assert not mismatches, f"{len(mismatches)} of {count} cases: " + "\n".join(mismatches[:20]) + + +def _reachability_chain_ends(world, waits): + """Return the mask of the ranks that every rank they reach through the waits reaches back.""" + reach = dict(waits) + changed = True + while changed: + changed = False + for r in range(world): + beyond = reach[r] + for q in range(world): + if reach[r] >> q & 1: + beyond |= reach[q] + changed |= beyond != reach[r] + reach[r] = beyond + ends = 0 + for r in range(world): + if all(reach[q] >> r & 1 for q in range(world) if reach[r] >> q & 1): + ends |= 1 << r + return ends + + +@pytest.mark.parametrize("world", [2, 3, 4]) +def test_chain_end_matches_reachability(world): + """Find the chain ends of every wait-for graph, cycles included. + + A rank is a chain end when every rank it reaches through the waits reaches it back. + """ + pairs = [(r, q) for r in range(world) for q in range(world) if q != r] + for edges in itertools.product((False, True), repeat=len(pairs)): + waits = {r: 0 for r in range(world)} + for (r, q), edge in zip(pairs, edges): + waits[r] |= edge << q + expected = _reachability_chain_ends(world, waits) + assert authority._chain_ends(range(world), waits) == expected, waits + + +@pytest.mark.parametrize("world", [8, 32, 64]) +@pytest.mark.parametrize("density", [0.02, 0.1, 0.5]) +def test_chain_ends_match_reachability_on_random_graphs(world, density): + """Find the chain ends of random wait-for graphs of many ranks, with chains and cycles.""" + rng = random.Random(world * 1000 + round(density * 100)) + for _ in range(20): + waits = { + r: sum(1 << q for q in range(world) if q != r and rng.random() < density) + for r in range(world) + } + expected = _reachability_chain_ends(world, waits) + assert authority._chain_ends(range(world), waits) == expected, waits + + +def test_hang_timer_restarts_after_stall_ends(): + """Time a hang from the start of the latest stall, not from an earlier stall that ended.""" + world, hung = 4, 2 + auth = Authority(world, hang_timeout_s=HANG_TIMEOUT_S) + everyone = (1 << world) - 1 + started = {r: RankReport(started=True) for r in range(world)} + assert auth.tick(0.0, started, everyone).state == MembershipState.COMMITTED + # Rank 2 makes no progress, and the others wait on it twice, first for less than the timeout. + first_stall, second_stall = range(1, 4), 6 + for tick in range(1, TICKS + 1): + waiting_for = 1 << hung if tick in first_stall or tick >= second_stall else 0 + reports = { + r: _report(received_s=tick * STEP_S, waiting_for=waiting_for) for r in range(world) + } + reports[hung] = _report(received_s=tick * STEP_S) + record = auth.tick(tick * STEP_S, reports, everyone) + if record is not None: + break + assert record is not None + assert record.active_mask == everyone & ~(1 << hung) + stalled_s = tick * STEP_S - second_stall * STEP_S + assert HANG_TIMEOUT_S < stalled_s <= HANG_TIMEOUT_S + 2 * STEP_S + + +@pytest.mark.parametrize("node_report", ["fresh", "stale"]) +def test_paused_rank_kept_only_while_node_report_fresh(node_report): + """Keep a paused rank that others wait on only while its node supervisor's report is fresh. + + Rank 2 stops reporting in a paused call. Its node supervisor's paused mask names it, but a + supervisor that stopped reporting no longer vouches for the pause after half the hang timeout. + The rank is then hung like a stopped rank, one hang timeout after its last report. + """ + world, paused = 4, 2 + auth = Authority(world, hang_timeout_s=HANG_TIMEOUT_S) + everyone = (1 << world) - 1 + started = {r: RankReport(started=True) for r in range(world)} + assert auth.tick(0.0, started, everyone).state == MembershipState.COMMITTED + record = None + for tick in range(1, TICKS + 1): + now_s = tick * STEP_S + reports = {r: _report(received_s=now_s, waiting_for=1 << paused) for r in range(world)} + reports[paused] = _report(received_s=STEP_S) + received_s = now_s if node_report == "fresh" else STEP_S + node = NodeReport(paused_mask=1 << paused, received_s=received_s) + record = auth.tick(now_s, reports, everyone, [node]) + if record is not None: + break + if node_report == "fresh": + assert record is None + else: + assert record.active_mask == everyone & ~(1 << paused) + assert HANG_TIMEOUT_S < now_s - STEP_S <= HANG_TIMEOUT_S + 2 * STEP_S + + +@pytest.mark.parametrize("after_pause", ["resumes", "stuck"]) +def test_hang_timer_starts_when_pause_ends(after_pause): + """Time a hang from the end of a pause, not from the last report sent before it. + + Rank 2 stops reporting in a paused call for two hang timeouts while the others wait on it. Its + node supervisor reports the pause ended one tick before the rank's own reports resume. A rank + that then resumes is kept, and one that stays stuck is hung one hang timeout after the pause. + """ + world, paused = 4, 2 + auth = Authority(world, hang_timeout_s=HANG_TIMEOUT_S) + everyone = (1 << world) - 1 + started = {r: RankReport(started=True) for r in range(world)} + assert auth.tick(0.0, started, everyone).state == MembershipState.COMMITTED + last_paused_tick = round(2 * HANG_TIMEOUT_S / STEP_S) + record = None + for tick in range(1, TICKS + 1): + now_s = tick * STEP_S + resumed = after_pause == "resumes" and tick > last_paused_tick + 1 + waiting_for = 0 if resumed else 1 << paused + reports = {r: _report(received_s=now_s, waiting_for=waiting_for) for r in range(world)} + reports[paused] = ( + _report(received_s=now_s, progress=tick) if resumed else _report(received_s=STEP_S) + ) + paused_mask = 1 << paused if tick <= last_paused_tick else 0 + node = NodeReport(paused_mask=paused_mask, received_s=now_s) + record = auth.tick(now_s, reports, everyone, [node]) + if record is not None: + break + if after_pause == "resumes": + assert record is None + else: + assert record.active_mask == everyone & ~(1 << paused) + stalled_s = now_s - last_paused_tick * STEP_S + assert HANG_TIMEOUT_S < stalled_s <= HANG_TIMEOUT_S + 2 * STEP_S diff --git a/tests/unittest/_torch/distributed/test_store_group.py b/tests/unittest/_torch/distributed/test_store_group.py new file mode 100644 index 000000000000..7179a5449e9b --- /dev/null +++ b/tests/unittest/_torch/distributed/test_store_group.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The collectives of ranks over the rendezvous store and the signing of its values. + +Ranks run as threads over an in-memory store. +""" + +import os +import pickle +import re +import threading + +import pytest +from _torch.distributed.store_group_test_utils import InMemoryStore, Unpickled +from mpi4py import MPI +from torch.distributed import TCPStore + +from tensorrt_llm._torch.distributed.membership import ( + SHARED_SECRET_ENV, + SignedStore, + StoreGroup, + UnsignedValueError, + rendezvous, +) +from tensorrt_llm._torch.distributed.membership.signed_store import shared_secret_digest + +pytestmark = pytest.mark.cpu_only + +JOIN_TIMEOUT_S = 10.0 +_SHORT_SECRET = f"shared secret of at least 32 bytes (64 hex digits) in {SHARED_SECRET_ENV}" + + +def _group(store, rank, ranks, **options): + return StoreGroup(store, "group", rank, ranks, **options) + + +def _run_ranks(calls): + """Run each rank's call on its own thread, and return its result or exception by rank.""" + results = {} + + def run(rank, call): + try: + results[rank] = call() + except Exception as error: + results[rank] = error + + threads = [threading.Thread(target=run, args=item, daemon=True) for item in calls.items()] + for thread in threads: + thread.start() + for thread in threads: + thread.join(JOIN_TIMEOUT_S) + assert not any(thread.is_alive() for thread in threads), "a rank did not return" + return results + + +def test_split_orders_members_by_key(): + """Check that a split orders its members by key, as MPI_Comm_split does.""" + store = InMemoryStore() + groups = {r: _group(store, r, range(4)) for r in range(4)} + + def split(rank): + subgroup = groups[rank].Split(color=rank % 2, key=-rank) + return subgroup.membership, subgroup.allgather(rank) + + results = _run_ranks({r: lambda r=r: split(r) for r in groups}) + assert results[0] == results[2] == ((2, 0), [2, 0]) + assert results[1] == results[3] == ((3, 1), [3, 1]) + + +def test_split_leaves_out_inactive_members(): + """Split the active members only, as an inactive member's color is unknown.""" + store = InMemoryStore() + active = (0, 1, 3) + groups = {r: _group(store, r, range(4), active=active) for r in active} + + def split(rank): + subgroup = groups[rank].Split(color=rank % 2, key=rank) + return subgroup.membership, subgroup.allgather(rank) + + results = _run_ranks({r: lambda r=r: split(r) for r in groups}) + assert results[0] == ((0,), [0]) + assert results[1] == results[3] == ((1, 3), [1, 3]) + + +def test_split_type_groups_ranks_by_node(): + """Check that a shared-memory split holds the node's ranks, and other split types raise.""" + group = _group(InMemoryStore(), 3, range(6), gpus_per_node=2) + assert group.Split_type(MPI.COMM_TYPE_SHARED).membership == (2, 3) + with pytest.raises(NotImplementedError): + group.Split_type(MPI.COMM_TYPE_SHARED + 1000) + + +@pytest.mark.parametrize("op, expected", [("SUM", 6), ("MIN", 1), ("MAX", 3)]) +def test_allreduce_over_active_members(op, expected): + """Reduce over the active members' values only.""" + store = InMemoryStore() + values = {0: 3, 1: 1, 3: 2} + groups = {r: _group(store, r, range(4), active=values) for r in values} + results = _run_ranks( + {r: lambda r=r: groups[r].allreduce(values[r], getattr(MPI, op)) for r in groups} + ) + assert results == {r: expected for r in values} + + +@pytest.mark.parametrize("root", [2, 1], ids=["active", "inactive"]) +def test_bcast_over_active_members(root): + """Broadcast the root's object to the active members, or None from an inactive root.""" + store = InMemoryStore() + active = (0, 2, 3) + groups = {r: _group(store, r, range(4), active=active) for r in active} + results = _run_ranks({r: lambda r=r: groups[r].bcast(f"from {r}", root) for r in groups}) + assert results == {r: f"from {root}" if root in active else None for r in active} + + +def test_on_wait_error_fails_exchange(): + """Check that an error raised by ``on_wait`` ends the exchange and marks it failed.""" + waiting = [] + + def on_wait(mask): + waiting.append(mask) + if mask: + raise TimeoutError("rank 1 stopped") + + group = _group(InMemoryStore(), 0, (0, 1), on_wait=on_wait) + with pytest.raises(TimeoutError): + group.allgather(0) + assert waiting == [0b10, 0] + assert group.exchange_failed + + +def test_exchange_failed_tells_latest_exchange(): + """Check that ``exchange_failed`` tells whether the latest exchange raised.""" + failing = [True] + + def on_wait(mask): + if failing[0]: + raise TimeoutError("stopped waiting") + + group = _group(InMemoryStore(), 0, (0,), on_wait=on_wait) + with pytest.raises(TimeoutError): + group.allgather(0) + assert group.exchange_failed + failing[0] = False + assert group.allgather(1) == [1] + assert not group.exchange_failed + + +def test_exchange_keeps_only_latest_call_keys(): + """Delete a call's keys once the next call completes, so only the latest call's keys stay.""" + store = InMemoryStore() + groups = {r: _group(store, r, (0, 1)) for r in (0, 1)} + _run_ranks({r: lambda r=r: [groups[r].allgather(call) for call in range(3)] for r in groups}) + assert sorted(store.values) == ["group/2/0", "group/2/1", "group/2/arrived/0"] + + +def test_waiting_mask_beyond_first_counter(monkeypatch): + """Report a missing rank 64, whose arrival bit is in the second counter.""" + monkeypatch.setattr(rendezvous, "REPORT_INTERVAL_S", 0.01) + store = InMemoryStore() + waiting = {0: [], 63: []} + rank_63_waits = threading.Event() + + def on_wait(rank): + def report(mask): + waiting[rank].append(mask) + if rank == 63: + rank_63_waits.set() + if mask == 1 << 64: + raise TimeoutError("rank 64 stopped") + + return report + + groups = {r: _group(store, r, (0, 63, 64), on_wait=on_wait(r)) for r in (0, 63)} + results = _run_ranks( + { + 0: lambda: rank_63_waits.wait(10) and groups[0].allgather(0), + 63: lambda: groups[63].allgather(63), + } + ) + assert all(isinstance(result, TimeoutError) for result in results.values()) + assert waiting[0] == [1 << 64, 0] + + +def test_store_client_requires_set_secret(monkeypatch): + """Refuse a store client and a secret digest without a shared secret, naming the variable.""" + monkeypatch.delenv(SHARED_SECRET_ENV, raising=False) + with pytest.raises(ValueError, match=re.escape(_SHORT_SECRET)): + SignedStore(InMemoryStore()) + with pytest.raises(ValueError, match=re.escape(_SHORT_SECRET)): + shared_secret_digest(b"label") + + +@pytest.mark.parametrize("secret", ["ab" * 31, "zz" * 32], ids=["short", "not_hex"]) +def test_store_client_requires_ranks_secret(secret): + """Refuse a store client, as of the node supervisor or the authority, with a weak secret.""" + with pytest.raises(ValueError, match=re.escape(_SHORT_SECRET)): + SignedStore(InMemoryStore(), secret) + + +@pytest.mark.parametrize("forger_secret", [None, "cd" * 32], ids=["unsigned", "other_secret"]) +def test_forged_store_value_is_never_unpickled(monkeypatch, forger_secret): + """Reject a pickle that a process without the deployment's secret set, before unpickling it.""" + store = InMemoryStore() + forger = store if forger_secret is None else SignedStore(store, forger_secret) + signed = SignedStore(store, "ab" * 32) + monkeypatch.setattr(Unpickled, "calls", []) + forged = pickle.dumps(Unpickled()) + # Rank 1's value and arrival at the first call of a group of ranks 0 and 1. + forger.set("forged/0/1", forged) + forger.add("forged/0/arrived/0", 1 << 1) + with pytest.raises(ValueError, match="is not signed with this deployment's"): + StoreGroup(signed, "forged", 0, [0, 1]).allgather("rank 0") + assert Unpickled.calls == [] + # The deployment's own values pass. + signed.set("own", forged) + assert signed.get("own") == forged + + +def _libuv_store(): + """Host a TCPStore on torch's default server backend, whose values are limited to 8 MiB.""" + return TCPStore("127.0.0.1", 0, is_master=True, wait_for_workers=False, use_libuv=True) + + +def test_value_over_store_server_limit(): + """Set, get and delete values over the 8 MiB that torch's default TCPStore server accepts.""" + server = _libuv_store() + keys = server.num_keys() + signed = SignedStore(server, "ab" * 32) + large = os.urandom(9 << 20) + signed.set("large", large) + signed.set("small", b"small") + assert signed.get("large") == large + assert signed.multi_get(["small", "large"]) == [b"small", large] + assert signed.delete_key("large") and signed.delete_key("small") + assert server.num_keys() == keys + + +def test_forged_chunks_rejected(): + """Reject a large value whose chunks another process set under the deployment's signed key.""" + server = _libuv_store() + signed = SignedStore(server, "ab" * 32) + signed.set("large", os.urandom(9 << 20)) + deployment_key = server.get("large") + SignedStore(server, "cd" * 32).set("large", os.urandom(9 << 20)) + server.set("large", deployment_key) + with pytest.raises(UnsignedValueError, match="do not match its signed digest"): + signed.get("large") From 1c0896335ba6bb73a4ec13ad8b3dc0b676009e5d Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 00:16:54 -0800 Subject: [PATCH 07/13] [None][fix] Mark the fault-tolerance shared-secret variable name for bandit SHARED_SECRET_ENV holds the name of an environment variable, not a secret; bandit's B105 check flags it. Signed-off-by: Andrew Rusli --- tensorrt_llm/_wide_ep_fault_tolerance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/_wide_ep_fault_tolerance.py b/tensorrt_llm/_wide_ep_fault_tolerance.py index 52dd8218556c..8744c0dccee5 100644 --- a/tensorrt_llm/_wide_ep_fault_tolerance.py +++ b/tensorrt_llm/_wide_ep_fault_tolerance.py @@ -17,6 +17,6 @@ # The rendezvous store's host:port. The authority hosts the store. RENDEZVOUS_ENV = "TLLM_FAULT_TOLERANCE_RENDEZVOUS" # The hex secret that every deployment process holds. It derives the hello token. -SHARED_SECRET_ENV = "TLLM_FAULT_TOLERANCE_SHARED_SECRET" +SHARED_SECRET_ENV = "TLLM_FAULT_TOLERANCE_SHARED_SECRET" # nosec B105 - environment variable name, not a secret # The alternative that each error about an unmet fault tolerance requirement offers. TURN_FAULT_TOLERANCE_OFF = "turn fault tolerance off (enable_wide_ep_fault_tolerance=False)" From be180d2b3a924b099f7af41adbb1d4d776168e94 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 04:41:26 -0800 Subject: [PATCH 08/13] [None][fix] Reload the slots that a stopped expert restore wrote A rejoin restore can overwrite occupied slots, and a newer membership change can stop it before it installs its placement. A later plan that kept such a slot's old expert skipped it and then cleared the invalid slots, leaving the stopped restore's weights there. current_placement() returns the installed placement with this rank's invalid slots set to -1; an active rank's row of the plan's current placement is that view, so the plan loads those slots again on every rank alike. Signed-off-by: Andrew Rusli --- .../moe/fused_moe/eplb_expert_restore.py | 24 ++++++++++++++++--- .../_torch/moe/test_moe_load_balancer.py | 24 +++++++++++++++++++ tests/unittest/_torch/moe/test_spare_slots.py | 11 +++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py index df1f0abbe5e6..18b4f0ed7a72 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py +++ b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py @@ -287,6 +287,22 @@ def _recoverable_failures(nodes: list[int]) -> list[list[int]]: rank_0_node = node_ranks.pop(nodes[0], []) return list(node_ranks.values()) + [[rank] for rank in rank_0_node[1:]] + def current_placement(self) -> list[tuple[int, list[list[int]]]]: + """Return each layer's expert count and placement as this rank sees it.""" + return [ + (layer.expert_count, self._placement_view(i, layer).tolist()) + for i, layer in enumerate(self.balancer.single_layer_load_balancers) + ] + + def _placement_view(self, layer_idx: int, layer: SingleLayerMoeLoadBalancer) -> np.ndarray: + """Return ``layer``'s placement as this rank sees it, with no expert in invalid slots. + + A restore that stopped before it installed its placement may have written part of a slot. + """ + placement = np.array(layer.get_old_rank_expert_ids(), dtype=np.int32) + placement[self.balancer.ep_rank, sorted(self._invalid_slots.get(layer_idx, ()))] = -1 + return placement + def plan_expert_restore( self, active_ranks: Sequence[bool], @@ -298,7 +314,8 @@ def plan_expert_restore( Args: active_ranks: Whether each EP rank is active. current: The per-layer placement that the active ranks agreed on, so that every - active rank plans the same placement. + active rank plans the same placement. An active rank's row is its row of + ``current_placement``, so a slot that a stopped restore wrote loads again. rejoining: Whether each EP rank rejoins. If one does, the active slots of all active ranks take the initial placement, which the rejoining rank loaded. @@ -416,8 +433,9 @@ def restore_lost_experts( plan: The plan that every active rank restores. times: Receives each phase's duration in milliseconds. progress: Called after the backing and each layer loaded, so that a long restore keeps - reporting progress. It may raise to stop the restore. The slots loaded so far then - hold no expert in this rank's view until the next restore. + reporting progress. It may raise to stop the restore. The slots that ``plan`` loads + on this rank then hold no expert in ``current_placement`` until a restore installs a + placement. Raises: RuntimeError: An iteration is active, or the online EPLB worker may still write the diff --git a/tests/unittest/_torch/moe/test_moe_load_balancer.py b/tests/unittest/_torch/moe/test_moe_load_balancer.py index f89621f3931d..e8ece313e720 100644 --- a/tests/unittest/_torch/moe/test_moe_load_balancer.py +++ b/tests/unittest/_torch/moe/test_moe_load_balancer.py @@ -800,6 +800,30 @@ def assert_loaded(expert_ids): balancer.abandon_iterations() if layer_updates_per_iter: assert layer.get_old_rank_expert_ids()[0] == [3, 0, 1, 2, -1, -1] + # A newer membership change stops a rejoin restore once it loaded + # rank 0's active slots with the initial placement, before it + # installed that placement. + stopped = balancer.expert_restore.plan_expert_restore( + [True, True], [layer.get_old_rank_expert_ids()], + rejoining=[False, True]) + with pytest.raises(RuntimeError, match="newer membership change"): + balancer.expert_restore.restore_lost_experts( + stopped, + progress=MagicMock(side_effect=[ + None, RuntimeError("a newer membership change") + ])) + assert layer.get_old_rank_expert_ids()[0] == [3, 0, 1, 2, -1, -1] + assert_loaded([0, 1, 2]) + # Rank 1 fails again. Rank 0's view names no expert in the slots + # the stopped restore wrote, so the next plan loads them again. + ((_, view), ) = balancer.expert_restore.current_placement() + retry = balancer.expert_restore.plan_expert_restore([True, False], + [view]) + balancer.expert_restore.restore_lost_experts(retry) + assert_loaded(layer.get_old_rank_expert_ids()[0][:4]) + assert layer.get_old_rank_expert_ids()[0] == [0, 1, 3, 2, -1, -1] + assert route([0, 1, 3, 2]) == [0, 1, 2, 3] + balancer.abandon_iterations() rejoin = balancer.expert_restore.plan_expert_restore( [True, True], [layer.get_old_rank_expert_ids()], rejoining=[False, True]) diff --git a/tests/unittest/_torch/moe/test_spare_slots.py b/tests/unittest/_torch/moe/test_spare_slots.py index 34a0952c0178..cbc97315a7c1 100644 --- a/tests/unittest/_torch/moe/test_spare_slots.py +++ b/tests/unittest/_torch/moe/test_spare_slots.py @@ -346,6 +346,17 @@ def test_restore_reads_no_placement_while_an_update_may_swap_it(monkeypatch): old_rank_expert_ids.assert_not_called() +def test_current_placement_hides_invalid_slots(monkeypatch): + """Show no expert in a slot of this rank that a stopped restore may have written in part.""" + load_balancer, _ = _online_balancer(monkeypatch, _ONLINE, ep_rank=2) + restore = load_balancer.expert_restore + assert restore.current_placement() == [(EXPERTS, _ONLINE.tolist())] + restore._invalid_slots[0] = {1} + expected = _ONLINE.copy() + expected[2, 1] = -1 + assert restore.current_placement() == [(EXPERTS, expected.tolist())] + + class _SpareSlotOwner(MoEWeightOwnerMixin, MoEEplbWeightLayoutMixin): """A MoE layer's weight owner with the quantization method of the MoE backend ``leaf``.""" From d3470947d67bc5354d9a14b81812cbef785a066c Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 18:47:20 -0800 Subject: [PATCH 09/13] [None][fix] Release only the spare slots that the installed placement leaves empty A placement can leave an active slot empty (a restore stopped after it loaded a survivor's active slots), and the load balancer reports every empty slot as unusable. Releasing them all passed the active slot to the unbacked-slot memory, which refuses it, so nothing was released. Release only the unusable slots that are spare slots. Signed-off-by: Andrew Rusli --- .../moe/fused_moe/eplb_expert_restore.py | 8 ++- .../_torch/moe/test_moe_load_balancer.py | 63 +++++++++++++++++++ tests/unittest/_torch/moe/test_spare_slots.py | 6 +- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py index 18b4f0ed7a72..1ecb3e37c52d 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py +++ b/tensorrt_llm/_torch/moe/fused_moe/eplb_expert_restore.py @@ -405,7 +405,8 @@ def set_spare_slots_backed(self, layer_slots: Sequence[Sequence[int]], backed: b def release_empty_spare_slots(self) -> int: """Map the spare slots that the installed placement leaves empty back to the shared scratch. - An online EPLB update may still run, because it never fills these slots. + An online EPLB update may still run, because it never fills these slots. An active slot + that a stopped restore emptied is unusable too, but it has no unbacked memory to release. Returns: The number of bytes released. @@ -414,7 +415,10 @@ def release_empty_spare_slots(self) -> int: if balancer.in_iter: raise RuntimeError("Cannot release spare slots while an iteration is active") return self.set_spare_slots_backed( - [layer.get_unusable_local_slots() for layer in balancer.single_layer_load_balancers], + [ + [slot for slot in layer.get_unusable_local_slots() if slot in layer.spare_slots] + for layer in balancer.single_layer_load_balancers + ], False, ) diff --git a/tests/unittest/_torch/moe/test_moe_load_balancer.py b/tests/unittest/_torch/moe/test_moe_load_balancer.py index e8ece313e720..337e28223311 100644 --- a/tests/unittest/_torch/moe/test_moe_load_balancer.py +++ b/tests/unittest/_torch/moe/test_moe_load_balancer.py @@ -860,6 +860,69 @@ def assert_loaded(expert_ids): balancer.shutdown() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +def test_release_skips_empty_active_slot(monkeypatch): + """Release an empty spare slot while an active slot is empty too. + + The empty active slot is unusable as well, but it has no unbacked memory to release. + """ + monkeypatch.setenv("TLLM_HOST_ACCESSIBLE_ALLOW_MANAGED_FALLBACK", "1") + # Two ranks on two nodes, so rank 0 gets three spare slots after EPLB's three. + monkeypatch.setattr(EplbExpertRestore, "_rank_nodes", lambda self: [0, 1]) + torch.cuda.set_device(0) + expert_count = 4 + # Slots of 8 MiB, whole VMM chunks. + slot_elements = 1 << 21 + balancer = FaultTolerantMoeLoadBalancer( + 0, + 2, + 0, + shared_memory_base_name=f"test_release_{os.getpid()}", + mapping=None) + try: + layer = balancer.add_layer(expert_count, 1, 3) + slot_count = layer.total_slot_count_per_rank + active = layer.active_slot_count_per_rank + assignments = layer.with_spare_slots([0, 1, 2, 3, 0, 1]) + host = [torch.randn(slot_elements) for _ in range(expert_count)] + loaded = torch.stack([host[e] for e in assignments[:active]] + + [torch.zeros(slot_elements)] * + (slot_count - active)) + weights = create_spare_slot_storage(layer, "w", loaded) + assert weights is not None + for expert_id in range(expert_count): + layer.host_tensor_sharer.share_host_tensor_with_shape( + expert_id, "w", host[expert_id]) + layer.add_to_migrate_weight_fn(use_spare_slot_storage, + (layer, weights, "w")) + layer.host_tensor_sharer.finalize_layer_weights() + layer.set_initial_weight_assignments(assignments) + balancer.register_weight_slots_after_to_cuda() + balancer.finalize_model() + balancer.set_warm_up_iter_count(0) + restore = balancer.expert_restore + # Rank 1 fails: its expert 3 moves into rank 0's spare slot 3, which a + # restore backs. + plan = restore.plan_expert_restore([True, False], + [layer.get_old_rank_expert_ids()]) + assert restore.restore_lost_experts(plan) == plan.device_bytes > 0 + # Rank 1 is back with a placement that leaves rank 0's active slot 0 + # empty, as a stopped restore leaves a slot whose expert has another + # replica. + balancer.load_balancer_impl.restore_placement( + [], [[[-1, 1, 2, -1, -1, -1], [3, 0, 1, -1, -1, -1]]]) + assert layer.get_unusable_local_slots() == [0, 3, 4, 5] + free = torch.cuda.mem_get_info()[0] + assert restore.release_empty_spare_slots() == plan.device_bytes + assert restore.backed_spare_slot_bytes == 0 + # At most the shared scratch chunk stays mapped. + assert torch.cuda.mem_get_info()[0] - free >= plan.device_bytes - ( + 2 << 20) + assert spare_slot_bytes_to_back(layer, [3]) > 0 + finally: + balancer.shutdown() + + _EXPERTS, _HIDDEN, _INTERMEDIATE = 4, 2048, 2048 diff --git a/tests/unittest/_torch/moe/test_spare_slots.py b/tests/unittest/_torch/moe/test_spare_slots.py index cbc97315a7c1..78c09e0c93a1 100644 --- a/tests/unittest/_torch/moe/test_spare_slots.py +++ b/tests/unittest/_torch/moe/test_spare_slots.py @@ -314,11 +314,15 @@ def test_release_keeps_filled_spare_slots_during_update(monkeypatch): """Release only the empty spare slots while an online EPLB update swaps the placement.""" # The old placement reads -1 during the swap, so only the unusable slots tell 2 from 3. load_balancer, layer = _online_balancer(monkeypatch, _placement(*[[-1] * 4] * WORLD_SIZE)) - layer.single_layer_load_balancer_impl.get_unusable_local_slots.return_value = [3] + # Active slot 1 is unusable too: a stopped restore emptied it, and another replica kept its + # expert, so the next plan left it empty. + layer.single_layer_load_balancer_impl.get_unusable_local_slots.return_value = [1, 3] released = [] def set_backed(tensors, slots, slot_counts, backed): assert not backed + # Like moe_unbacked_slots_set_backed, refuse a slot that is not a spare slot. + assert all(slot in layer.spare_slots for slot in slots), slots released.extend(slots) return len(slots) * (4 << 20) From f08dcfee859935259aa4eee8902514b2a6ba7c76 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 18:47:36 -0800 Subject: [PATCH 10/13] [None][fix] Count a joining rank's READY when finding ranks that did not reply The authority counts a joining rank's joining_ready report as its READY when it decides that every rank replied, but the hang check listed the ranks still waited on without it, so a joiner at READY could be removed as hung. Signed-off-by: Andrew Rusli --- .../distributed/membership/authority.py | 2 +- .../_torch/distributed/test_membership.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/distributed/membership/authority.py b/tensorrt_llm/_torch/distributed/membership/authority.py index 4b422eae40dd..ce9766cef007 100644 --- a/tensorrt_llm/_torch/distributed/membership/authority.py +++ b/tensorrt_llm/_torch/distributed/membership/authority.py @@ -544,7 +544,7 @@ def _detect_failures( waited_on = 0 if state in expected: waited_on = rank_mask( - (r for r in active if not self._replied(reports.get(r), expected[state])), + (r for r in active if not self._rank_replied(r, reports, expected[state])), self.world_size, ) # During the startup a rank that started waits on every rank that has not. diff --git a/tests/unittest/_torch/distributed/test_membership.py b/tests/unittest/_torch/distributed/test_membership.py index ddb8d24669d3..860dedae16e3 100644 --- a/tests/unittest/_torch/distributed/test_membership.py +++ b/tests/unittest/_torch/distributed/test_membership.py @@ -468,6 +468,29 @@ def test_stalled_joiner_left_out_early(): assert cluster.now - stalled_s <= admission_s + 0.2 +def test_ready_joiner_not_hung_while_survivor_finishes_step(): + """Keep a joiner whose readiness counts as READY while a survivor finishes its step. + + The joiner sends no READY of its own and makes no progress, so the authority must not wait on + it, or its admission timeout would leave it out. + """ + admission_s = 2.0 + cluster = Cluster(admission_timeout_s=admission_s) + cluster.alive.discard(2) + cluster.run(1.0) + _join(cluster, 2) + cluster.run_until(MembershipState.PROPOSE) + proposed = cluster.authority.record + cluster.hung.update({1, 2}) # neither replies READY + cluster.installing.add(1) # rank 1 still makes progress in its step + assert cluster.run(admission_s + 1.0) == proposed + _release(cluster) + cluster.installing.clear() + record = cluster.run(1.0) + assert record.state == MembershipState.COMMITTED + assert record.active_mask == ALL + + def test_frozen_survivor_removed_before_joiner(): """Remove a survivor that froze during an admission, not the joiner it holds up.""" cluster = Cluster(hang_timeout_s=4.0, admission_timeout_s=1.0) From fa400071e697817f650588b91fe71956cefa6ec6 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 18:47:37 -0800 Subject: [PATCH 11/13] [None][fix] Wait on liveness sockets with selectors select.select refuses file descriptors at or above FD_SETSIZE (1024); the membership client now waits through selectors. Signed-off-by: Andrew Rusli --- .../_torch/distributed/membership/liveness.py | 17 ++++++++-- .../_torch/distributed/test_membership.py | 31 ++++++++++++++++--- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/_torch/distributed/membership/liveness.py b/tensorrt_llm/_torch/distributed/membership/liveness.py index 4d9f870350b6..53beb2173ad7 100644 --- a/tensorrt_llm/_torch/distributed/membership/liveness.py +++ b/tensorrt_llm/_torch/distributed/membership/liveness.py @@ -16,7 +16,7 @@ import functools import os -import select +import selectors import socket import threading import time @@ -114,6 +114,17 @@ def close_in_forked_child(resource: socket.socket | IO[bytes]) -> None: _closed_in_forked_child().add(resource) +def _readable(watched: list, timeout_s: Optional[float]) -> list: + """Return the files of ``watched`` that are readable within ``timeout_s``, None to block. + + Unlike ``select.select``, this accepts descriptors numbered above ``FD_SETSIZE``. + """ + with selectors.DefaultSelector() as selector: + for fileobj in watched: + selector.register(fileobj, selectors.EVENT_READ) + return [key.fileobj for key, _ in selector.select(timeout_s)] + + def _warn_while_no_authority(store: SignedStore) -> None: """Log once if the authority has not published its liveness address within the warning time.""" from torch.distributed import DistStoreError @@ -372,14 +383,14 @@ def _serve(self) -> None: return listed = member watched = [self._sock, self._wake_read] - readable, _, _ = select.select(watched, [], [], max(0.0, tick_s - time.monotonic())) + readable = _readable(watched, max(0.0, tick_s - time.monotonic())) if ( self._sock not in readable and time.monotonic() - heard_s > AUTHORITY_SILENCE_TIMEOUT_S ): # This thread can wait long for a CPU or the GIL after its select returns, while the # authority's lines arrive. Look again before taking the authority as silent. - readable, _, _ = select.select(watched, [], [], 0.0) + readable = _readable(watched, 0.0) if self._sock not in readable: raise TimeoutError( "the membership authority sent nothing for " diff --git a/tests/unittest/_torch/distributed/test_membership.py b/tests/unittest/_torch/distributed/test_membership.py index 860dedae16e3..75131fbf5197 100644 --- a/tests/unittest/_torch/distributed/test_membership.py +++ b/tests/unittest/_torch/distributed/test_membership.py @@ -21,6 +21,7 @@ import logging import os import random +import resource import select import signal import socket @@ -1641,23 +1642,43 @@ def test_late_wakeup_keeps_live_authority(monkeypatch): liveness, "time", SimpleNamespace(monotonic=lambda: time.monotonic() + skipped_s[0]) ) calls = [] + readable = liveness._readable - def select_with_late_wakeup(watched, *rest): + def readable_with_late_wakeup(watched, timeout_s): calls.append(watched) if len(calls) > 1: - return select.select(watched, *rest) + return readable(watched, timeout_s) # The authority's line has arrived, but the thread wakes only after the silence timeout. - select.select(watched[:1], [], []) + readable(watched[:1], None) skipped_s[0] += 2 * silence_s - return [], [], [] + return [] - monkeypatch.setattr(liveness, "select", SimpleNamespace(select=select_with_late_wakeup)) + monkeypatch.setattr(liveness, "_readable", readable_with_late_wakeup) errors = [] with _scripted_client(errors, resend_s=0.1): _until(lambda: errors or len(calls) > 2) assert not errors +def test_client_serves_descriptors_above_select_limit(): + """Keep a client whose liveness socket and wake pipe are numbered above select's limit.""" + if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 2048: + pytest.skip("the open-file limit is below 2048") + placeholders = [] + try: + while not placeholders or placeholders[-1] < 1024: + placeholders.append(os.open(os.devnull, os.O_RDONLY)) + errors = [] + with _scripted_client(errors, resend_s=0.1) as client: + assert client._sock.fileno() > 1024 + time.sleep(1.0) + assert not errors + assert client._thread.is_alive() + finally: + for fd in placeholders: + os.close(fd) + + def test_callback_error_logged_with_traceback(monkeypatch): """Report an error of a client's callback with its traceback, not as a lost authority.""" logged, errors = [], [] From 2efb4420db72379c8d54404d6bbab993e5ffeb57 Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 18:47:37 -0800 Subject: [PATCH 12/13] [None][docs] State what the resumed epoch of a replaced INSTALL means for a rank that started a round Signed-off-by: Andrew Rusli --- tensorrt_llm/_torch/distributed/membership/authority.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/distributed/membership/authority.py b/tensorrt_llm/_torch/distributed/membership/authority.py index ce9766cef007..b229918c0e1d 100644 --- a/tensorrt_llm/_torch/distributed/membership/authority.py +++ b/tensorrt_llm/_torch/distributed/membership/authority.py @@ -374,8 +374,8 @@ def _resume_epoch(self, epochs: list[int]) -> int: replaced = self._replaced_install_epoch # A newer PROPOSE can replace an INSTALL before its commit, after some ranks moved to that # INSTALL's resume epoch. Every round waits for every active rank, so if a survivor is still - # before that epoch, no rank ran a round at it. Resume there again, which keeps the ranks - # that installed it consistent. + # before that epoch, no rank finished a round at it; a rank that started one stopped in it + # and runs it again. Resume there again, which keeps the ranks that installed it consistent. if replaced and any(latest_epoch([replaced, epoch]) != epoch for epoch in epochs): return replaced # Epochs wrap, so the resume epoch follows the latest one, not the largest. From 7a50994a0917593d37876c10ba3f25a5742932fe Mon Sep 17 00:00:00 2001 From: Andrew Rusli Date: Fri, 20 Nov 2026 19:02:20 -0800 Subject: [PATCH 13/13] [None][docs] State the spare-slot contract of a MoE backend A backend that declares supports_spare_slots keeps its slot-first expert weights in the spare-slot storage through loading, binds tuner_layout_kwargs into every tunable op and sizes runtime workspaces for every local slot. The fallback warning is logged once per requested backend and chosen class. Signed-off-by: Andrew Rusli --- tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md | 2 +- tensorrt_llm/_torch/moe/fused_moe/impl_contract.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md index 6d9ab45ffb2c..4f9d7efc54f8 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/moe/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -255,7 +255,7 @@ wakeup. Under `enable_wide_ep_fault_tolerance`, `CommunicationFactory` builds `NVLinkOneSided` with its rank-mask kernels, and the model loader binds every MoE layer's communication to one `MoeExecutionState` (`communication/execution_state.py`, bound by `pyexecutor/rank_failure_model_load.py`) before the weights load. The kernels exchange only with the EP ranks set in the state's device mask, which they read when they run, so captured CUDA graphs follow a mask that `install_mask` rewrites while no collective runs. They stop waiting on peers once the state's pinned host abort state is requested (`request_abort`), and a wait on a peer longer than about 10 ms publishes the round flag it waits for there (`waited_flags`). A collective that ends without some active peer marks its round incomplete in the workspace, so later collectives exchange with this rank only until `rearm` resets the workspaces. These layers set up no sentinel transport, whose kernels take no rank mask. -A MoE backend runs with the unbacked spare EPLB slots of fault tolerance only if it declares `MoEStaticCapability.supports_spare_slots`: its weight method creates the spare slots and its autotuner profiles only the active slots (`tuner_layout_kwargs`). The Cutlass unquantized and NVFP4 leaves and the CuteDSL NVFP4 grouped-GEMM leaf declare it. While the load balancer adds spare slots (`MoEDeployment.spare_slots_enabled`), selection turns down every other candidate that `can_implement` accepts with `MoERejectReason.SPARE_SLOTS_UNSUPPORTED`, as it does for EPLB, and so does the CuteDSL leaf's `can_implement` on SM107, whose kernels tune every local slot. A backend literal then falls back like any degradation, to a Cutlass leaf with the capability, but the warning, which names Wide-EP fault tolerance and the reason, is logged once per process rather than once per layer; a pinned tactic, or a layer whose format no leaf with the capability serves, fails with that reason. +A MoE backend runs with the unbacked spare EPLB slots of fault tolerance only if it declares `MoEStaticCapability.supports_spare_slots` and keeps this contract: every expert weight has the slot dimension first, so that `MoEWeightOwnerMixin.create_weights` moves it to the spare-slot storage (`create_spare_slot_weights`); the backend keeps that storage in place through loading and every post-load transform, or `use_spare_slot_storage` raises; every tunable op binds `tuner_layout_kwargs`, so that the autotuner profiles only the active slots; and runtime workspaces cover every local slot, since a restore routes tokens to spare slots. The Cutlass unquantized and NVFP4 leaves and the CuteDSL NVFP4 grouped-GEMM leaf declare it. While the load balancer adds spare slots (`MoEDeployment.spare_slots_enabled`), selection turns down every other candidate that `can_implement` accepts with `MoERejectReason.SPARE_SLOTS_UNSUPPORTED`, as it does for EPLB, and so does the CuteDSL leaf's `can_implement` on SM107, whose kernels tune every local slot. A backend literal then falls back like any degradation, to a Cutlass leaf with the capability, but the warning, which names Wide-EP fault tolerance and the reason, is logged once per requested backend and chosen class rather than once per layer; a pinned tactic, or a layer whose format no leaf with the capability serves, fails with that reason. ### Core Design Principles diff --git a/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py b/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py index b32a43a3c4a3..268a0d377785 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py +++ b/tensorrt_llm/_torch/moe/fused_moe/impl_contract.py @@ -53,8 +53,9 @@ class MoEStaticCapability: # weights may be published concurrently with shared-expert computation. supports_per_layer_dynamic_eplb: bool = False # Whether the backend runs with the unbacked spare EPLB slots of Wide-EP - # fault tolerance: its autotuner profiles and its workspaces cover only the - # active slots. + # fault tolerance: its expert weights keep their slot-first storage through + # loading, its autotuner profiles only the active slots, and its runtime + # workspaces cover every local slot. supports_spare_slots: bool = False # Legacy gate: the ``assert moe_cls in [...]`` bias allow-list in # ``create_moe_backend``. Per-expert FC bias from the checkpoint, added