Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions cpp/tensorrt_llm/kernels/moe/trtllmGen/runner.cu
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <tensorrt_llm/common/assert.h>
#include <tensorrt_llm/common/envUtils.h>

#include <algorithm>

TRTLLM_NAMESPACE_BEGIN

namespace kernels
Expand Down Expand Up @@ -493,9 +495,13 @@ void Runner::run(void* hiddenState, void* hiddenStateScale, void* weights, void*
int32_t numExperts, int32_t numTokens, int32_t* permutedIdxToTokenIdx, int32_t* ptrNumNonExitingCtas,
int32_t* ptrTotalNumPaddedTokens, int32_t* ptrCtaIdxXyToBatchIdx, int32_t* ptrCtaIdxXyToMnLimit,
void* bmm1Workspace, bool useRoutingScalesOnInput, int device, cudaStream_t stream, int32_t configIndex,
int32_t validHiddenSize, int32_t validIntermediateSize)
int32_t validHiddenSize, int32_t validIntermediateSize, int32_t maxNumCtasInBatchDim)
{
auto maxNumCgasInBatchDim = Routing::getMaxNumCgasInBatchDim(numTokens, topK, numExperts, mTileTokensDim);
if (maxNumCtasInBatchDim > 0)
{
maxNumCgasInBatchDim = std::min(maxNumCgasInBatchDim, maxNumCtasInBatchDim);
}
bool is_gated_activation = tensorrt_llm::kernels::isGatedActType(mActType);
int32_t intermediateSizeFactor = (is_gated_activation ? 2 : 1);
mRunner.run(numTokens, intermediateSizeFactor * intermediateSize, hiddenSize, numTokens,
Expand Down Expand Up @@ -590,9 +596,13 @@ void Runner::run(void* permutedHiddenState, void* permutedHiddenStateScale, void
int32_t intermediateSize, int32_t numExperts, int32_t numTokens, int32_t* ptrNumNonExitingCtas,
int32_t* ptrTotalNumPaddedTokens, int32_t* ptrCtaIdxXyToBatchIdx, int32_t* ptrCtaIdxXyToMnLimit,
void* bmm2Workspace, int device, cudaStream_t stream, int32_t configIndex, int32_t validHiddenSize,
int32_t validIntermediateSize)
int32_t validIntermediateSize, int32_t maxNumCtasInBatchDim)
{
auto maxNumCgasInBatchDim = Routing::getMaxNumCgasInBatchDim(numTokens, topK, numExperts, mTileTokensDim);
if (maxNumCtasInBatchDim > 0)
{
maxNumCgasInBatchDim = std::min(maxNumCgasInBatchDim, maxNumCtasInBatchDim);
}
mRunner.run(numTokens, hiddenSize, intermediateSize, numTokens, validHiddenSize, validIntermediateSize, {},
numTokens, numExperts, maxNumCgasInBatchDim, permutedHiddenState, permutedHiddenStateScale, weights,
weightsScale, /* perTokensSfA */ nullptr,
Expand Down Expand Up @@ -842,7 +852,7 @@ void Runner::run(
workspace.cta_idx_xy_to_batch_idx, workspace.cta_idx_xy_to_mn_limit, workspace.bmm1_workspace,
args.mUseRoutingScalesOnInput, device, stream, config.gemm1Config,
args.valid_hidden_size.value_or(args.hidden_size),
args.valid_intermediate_size.value_or(args.intermediate_size));
args.valid_intermediate_size.value_or(args.intermediate_size), args.max_num_ctas_in_batch_dim);

// We do not fuse activation with FC1 for DeepSeek FP8 due to the weights shuffling constraint.
TLLM_CHECK_WITH_INFO(!args.skip_gemm2 || !args.do_finalize, "skip_gemm2 requires do_finalize == false.");
Expand All @@ -869,7 +879,7 @@ void Runner::run(
workspace.num_non_exiting_ctas, workspace.total_num_padded_tokens, workspace.cta_idx_xy_to_batch_idx,
workspace.cta_idx_xy_to_mn_limit, workspace.bmm2_workspace, device, stream, config.gemm2Config,
args.valid_hidden_size.value_or(args.hidden_size),
args.valid_intermediate_size.value_or(args.intermediate_size));
args.valid_intermediate_size.value_or(args.intermediate_size), args.max_num_ctas_in_batch_dim);

// Run finalize
if (args.do_finalize)
Expand Down
8 changes: 6 additions & 2 deletions cpp/tensorrt_llm/kernels/moe/trtllmGen/runner.h
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ class Runner
int32_t intermediateSize, int32_t numExperts, int32_t numTokens, int32_t* permutedIdxToTokenIdx,
int32_t* ptrNumNonExitingCtas, int32_t* ptrTotalNumPaddedTokens, int32_t* ptrCtaIdxXyToBatchIdx,
int32_t* ptrCtaIdxXyToMnLimit, void* bmm1Workspace, bool useRoutingScalesOnInput, int device,
cudaStream_t stream, int32_t configIndex, int32_t validHiddenSize = -1, int32_t validIntermediateSize = -1);
cudaStream_t stream, int32_t configIndex, int32_t validHiddenSize = -1, int32_t validIntermediateSize = -1,
int32_t maxNumCtasInBatchDim = 0);

private:
int32_t mTileTokensDim;
Expand Down Expand Up @@ -266,7 +267,7 @@ class Runner
int32_t intermediateSize, int32_t numExperts, int32_t numTokens, int32_t* ptrNumNonExitingCtas,
int32_t* ptrTotalNumPaddedTokens, int32_t* ptrCtaIdxXyToBatchIdx, int32_t* ptrCtaIdxXyToMnLimit,
void* bmm2Workspace, int device, cudaStream_t stream, int32_t configIndex, int32_t validHiddenSize = -1,
int32_t validIntermediateSize = -1);
int32_t validIntermediateSize = -1, int32_t maxNumCtasInBatchDim = 0);

private:
int32_t mTileTokensDim;
Expand Down Expand Up @@ -355,6 +356,9 @@ struct MoERunnerArgs
// Stop after FC1 (+ activation): the caller runs FC2 on workspace.gemm1_output / gemm1_output_scale with the
// routing tables. Requires do_finalize == false.
bool skip_gemm2{false};

// The most CTAs the caller's routing puts in the batch dimension, or 0 for the worst case of the problem.
int32_t max_num_ctas_in_batch_dim{0};
};

struct MoEWorkspace
Expand Down
115 changes: 100 additions & 15 deletions cpp/tensorrt_llm/thop/moe/trtllmGen/fp4BlockScaleMoe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ std::vector<torch::Tensor> run_fp4_block_scale_moe_runner(torch::optional<torch:
bool const do_finalize, btg::Dtype const dtype, MoeRunnerType& moe_runner, int64_t const moeConfigIndex,
torch::optional<torch::Tensor> const& topk_weights, torch::optional<torch::Tensor> const& topk_ids,
torch::optional<torch::Tensor> const& out_tensor = torch::nullopt, PreRoutedBuffers const* prerouted = nullptr,
bool const fc1_only = false)
bool const fc1_only = false, int64_t const prerouted_max_ctas = 0)
{
// fc1_only: routing (unless pre-routed) and FC1 only; the caller runs FC2 into the returned gemm2 output, which
// keeps the do_finalize == false contract (gemm2 output, expert weights, expanded -> permuted index).
Expand Down Expand Up @@ -230,6 +230,12 @@ std::vector<torch::Tensor> run_fp4_block_scale_moe_runner(torch::optional<torch:
= tensorrt_llm::kernels::trtllmGenFp8BlockScaleMoe::Routing::getRouteMapAllocCount(max_num_padded_tokens);
int32_t const max_num_ctas = tensorrt_llm::kernels::trtllmGenFp8BlockScaleMoe::Routing::getMaxNumCtasInBatchDim(
args.num_tokens, args.top_k, args.local_num_experts, tile_tokens_dim);
// Pre-routed tables may come with a smaller bound on their CTAs: the GEMMs launch that many (buffers keep the
// worst case), so fewer CTAs exit unused.
if (prerouted != nullptr && prerouted_max_ctas > 0)
{
args.max_num_ctas_in_batch_dim = static_cast<int32_t>(std::min<int64_t>(max_num_ctas, prerouted_max_ctas));
}
if (prerouted != nullptr)
{
// The kernels launch on the device of hidden_states and read these buffers directly.
Expand Down Expand Up @@ -553,24 +559,19 @@ std::vector<torch::Tensor> run_fp4_block_scale_moe_runner(torch::optional<torch:
auto const& moe_stream = at::cuda::getCurrentCUDAStream(hidden_states.get_device());
moe_runner.run(args, workspace, hidden_states.get_device(), moe_stream, moeConfigIndex);

// Pre-routed: the routing kernel that fills expert_weights did not run; the weights are topk_weights.
at::Tensor const routed_weights = prerouted != nullptr ? topk_weights.value() : expert_weights;
if (fc1_only)
{
return {gemm1_output, gemm1_output_scale, gemm2_output, routed_weights, expanded_idx_to_permuted_idx,
cta_idx_xy_to_batch_idx, cta_idx_xy_to_mn_limit, num_non_exiting_ctas};
}

if (!do_finalize)
{
// No output may alias an input: pre-routed, the permutation is the caller's table, which an empty
// [0, tile] placeholder names, and expert_weights is left unwritten (the weights are topk_weights).
if (prerouted != nullptr)
auto const permutation = prerouted != nullptr
? at::empty({0, tile_tokens_dim}, expanded_idx_to_permuted_idx.options())
: expanded_idx_to_permuted_idx;
if (fc1_only)
{
return {
gemm2_output, expert_weights, at::empty({0, tile_tokens_dim}, expanded_idx_to_permuted_idx.options())};
return {gemm1_output, gemm1_output_scale, gemm2_output, expert_weights, permutation,
cta_idx_xy_to_batch_idx, cta_idx_xy_to_mn_limit, num_non_exiting_ctas};
}
return {gemm2_output, expert_weights, expanded_idx_to_permuted_idx};
return {gemm2_output, expert_weights, permutation};
}

return {output};
Expand Down Expand Up @@ -722,6 +723,85 @@ class FP4BlockScaleMoeRunner : public torch::CustomClassHolder
/*fc1_only=*/false);
}

// runRoutingTables() up to FC1 + activation, with the arguments and outputs of runFc1.
[[nodiscard]] std::vector<torch::Tensor> runRoutingTablesFc1(torch::optional<torch::Tensor> const& routing_logits,
torch::optional<torch::Tensor> const& routing_bias, torch::Tensor const& hidden_states,
torch::Tensor const& hidden_states_scale, torch::Tensor const& gemm1_weights,
torch::Tensor const& gemm1_weights_scale, std::optional<torch::Tensor> const& gemm1_bias,
std::optional<torch::Tensor> const& gemm1_alpha, std::optional<torch::Tensor> const& gemm1_beta,
std::optional<torch::Tensor> const& gemm1_clamp_limit, torch::Tensor const& gemm2_weights,
torch::Tensor const& gemm2_weights_scale, torch::Tensor const& output1_scales_scalar,
torch::Tensor const& output1_scales_gate_scalar, torch::Tensor const& output2_scales_scalar,
int64_t const num_experts, int64_t const top_k, std::optional<int64_t> const n_group,
std::optional<int64_t> const topk_group, int64_t const intermediate_size, int64_t const local_expert_offset,
int64_t const local_num_experts, std::optional<double> const routed_scaling_factor,
int64_t const routing_method_type, std::vector<int64_t> moeConfigIndex,
torch::optional<torch::Tensor> const& topk_weights, torch::optional<torch::Tensor> const& topk_ids,
torch::Tensor const& routing_tables, std::vector<int64_t> const& routing_table_tiles)
{
return runImpl(routing_logits, routing_bias, hidden_states, hidden_states_scale, gemm1_weights,
gemm1_weights_scale, gemm1_bias, gemm1_alpha, gemm1_beta, gemm1_clamp_limit, gemm2_weights,
gemm2_weights_scale, std::nullopt, output1_scales_scalar, output1_scales_gate_scalar, output2_scales_scalar,
num_experts, top_k, n_group, topk_group, intermediate_size, local_expert_offset, local_num_experts,
routed_scaling_factor, routing_method_type, /*do_finalize=*/false, moeConfigIndex, topk_weights, topk_ids,
torch::nullopt, /*prerouted=*/nullptr, /*routing_tile_n=*/-1, routing_tables, routing_table_tiles,
/*fc1_only=*/true);
}

// runRoutingTables() on tables that put at most routing_max_ctas CTAs in the batch dimension of any tile; the
// GEMMs launch that many instead of the worst case.
[[nodiscard]] std::vector<torch::Tensor> runBounded(torch::optional<torch::Tensor> const& routing_logits,
torch::optional<torch::Tensor> const& routing_bias, torch::Tensor const& hidden_states,
torch::Tensor const& hidden_states_scale, torch::Tensor const& gemm1_weights,
torch::Tensor const& gemm1_weights_scale, std::optional<torch::Tensor> const& gemm1_bias,
std::optional<torch::Tensor> const& gemm1_alpha, std::optional<torch::Tensor> const& gemm1_beta,
std::optional<torch::Tensor> const& gemm1_clamp_limit, torch::Tensor const& gemm2_weights,
torch::Tensor const& gemm2_weights_scale, std::optional<torch::Tensor> const& gemm2_bias,
torch::Tensor const& output1_scales_scalar, torch::Tensor const& output1_scales_gate_scalar,
torch::Tensor const& output2_scales_scalar, int64_t const num_experts, int64_t const top_k,
std::optional<int64_t> const n_group, std::optional<int64_t> const topk_group, int64_t const intermediate_size,
int64_t const local_expert_offset, int64_t const local_num_experts,
std::optional<double> const routed_scaling_factor, int64_t const routing_method_type, bool const do_finalize,
std::vector<int64_t> moeConfigIndex, torch::optional<torch::Tensor> const& topk_weights,
torch::optional<torch::Tensor> const& topk_ids, torch::optional<torch::Tensor> const& output,
torch::Tensor const& routing_tables, std::vector<int64_t> const& routing_table_tiles,
int64_t const routing_max_ctas)
{
return runImpl(routing_logits, routing_bias, hidden_states, hidden_states_scale, gemm1_weights,
gemm1_weights_scale, gemm1_bias, gemm1_alpha, gemm1_beta, gemm1_clamp_limit, gemm2_weights,
gemm2_weights_scale, gemm2_bias, output1_scales_scalar, output1_scales_gate_scalar, output2_scales_scalar,
num_experts, top_k, n_group, topk_group, intermediate_size, local_expert_offset, local_num_experts,
routed_scaling_factor, routing_method_type, do_finalize, moeConfigIndex, topk_weights, topk_ids, output,
/*prerouted=*/nullptr, /*routing_tile_n=*/-1, routing_tables, routing_table_tiles,
/*fc1_only=*/false, routing_max_ctas);
}

// runBounded() up to FC1 + activation, with the arguments and outputs of runFc1.
[[nodiscard]] std::vector<torch::Tensor> runBoundedFc1(torch::optional<torch::Tensor> const& routing_logits,
torch::optional<torch::Tensor> const& routing_bias, torch::Tensor const& hidden_states,
torch::Tensor const& hidden_states_scale, torch::Tensor const& gemm1_weights,
torch::Tensor const& gemm1_weights_scale, std::optional<torch::Tensor> const& gemm1_bias,
std::optional<torch::Tensor> const& gemm1_alpha, std::optional<torch::Tensor> const& gemm1_beta,
std::optional<torch::Tensor> const& gemm1_clamp_limit, torch::Tensor const& gemm2_weights,
torch::Tensor const& gemm2_weights_scale, torch::Tensor const& output1_scales_scalar,
torch::Tensor const& output1_scales_gate_scalar, torch::Tensor const& output2_scales_scalar,
int64_t const num_experts, int64_t const top_k, std::optional<int64_t> const n_group,
std::optional<int64_t> const topk_group, int64_t const intermediate_size, int64_t const local_expert_offset,
int64_t const local_num_experts, std::optional<double> const routed_scaling_factor,
int64_t const routing_method_type, std::vector<int64_t> moeConfigIndex,
torch::optional<torch::Tensor> const& topk_weights, torch::optional<torch::Tensor> const& topk_ids,
torch::Tensor const& routing_tables, std::vector<int64_t> const& routing_table_tiles,
int64_t const routing_max_ctas)
{
return runImpl(routing_logits, routing_bias, hidden_states, hidden_states_scale, gemm1_weights,
gemm1_weights_scale, gemm1_bias, gemm1_alpha, gemm1_beta, gemm1_clamp_limit, gemm2_weights,
gemm2_weights_scale, std::nullopt, output1_scales_scalar, output1_scales_gate_scalar, output2_scales_scalar,
num_experts, top_k, n_group, topk_group, intermediate_size, local_expert_offset, local_num_experts,
routed_scaling_factor, routing_method_type, /*do_finalize=*/false, moeConfigIndex, topk_weights, topk_ids,
torch::nullopt, /*prerouted=*/nullptr, /*routing_tile_n=*/-1, routing_tables, routing_table_tiles,
/*fc1_only=*/true, routing_max_ctas);
}

// run() with caller-computed routing tables for tile routing_tile_n (see PreRoutedBuffers), used in place of the
// routing kernel when the tactic's tile matches. EP problems are passed all-local (num_experts = local_num_experts,
// local_expert_offset = 0, -1 for other ranks' routes) with topk_ids / topk_weights and no routing_logits.
Expand Down Expand Up @@ -858,7 +938,7 @@ class FP4BlockScaleMoeRunner : public torch::CustomClassHolder
torch::optional<torch::Tensor> const& topk_ids, torch::optional<torch::Tensor> const& output,
PreRoutedBuffers const* prerouted, int64_t const routing_tile_n,
torch::optional<torch::Tensor> const& routing_tables, std::vector<int64_t> const& routing_table_tiles,
bool const fc1_only)
bool const fc1_only, int64_t const routing_max_ctas = 0)
{
int64_t tileN = moeConfigIndex[0];
int64_t config = moeConfigIndex[1];
Expand Down Expand Up @@ -895,6 +975,7 @@ class FP4BlockScaleMoeRunner : public torch::CustomClassHolder
// routing_tables [tiles, 5, width]: per tile the PreRoutedBuffers rows, the last holding (num_non_exiting_ctas,
// total_num_padded_tokens). Only this tactic's tile is used; other tiles route as usual.
std::optional<PreRoutedBuffers> tables;
int64_t tables_max_ctas = 0;
if (routing_tables.has_value())
{
auto const& all = routing_tables.value();
Expand All @@ -907,14 +988,15 @@ class FP4BlockScaleMoeRunner : public torch::CustomClassHolder
auto const t = all.select(0, std::distance(routing_table_tiles.begin(), it));
tables = PreRoutedBuffers{t[0], t[1], t[2], t[3], t[4].narrow(0, 0, 1), t[4].narrow(0, 1, 1)};
prerouted = &tables.value();
tables_max_ctas = routing_max_ctas;
}
}
return run_fp4_block_scale_moe_runner(routing_logits, routing_bias, hidden_states, hidden_states_scale,
gemm1_weights, gemm1_weights_scale, gemm1_bias, gemm1_alpha, gemm1_beta, gemm1_clamp_limit, gemm2_weights,
gemm2_weights_scale, gemm2_bias, output1_scales_scalar, output1_scales_gate_scalar, output2_scales_scalar,
num_experts, top_k, n_group, topk_group, intermediate_size, local_expert_offset, local_num_experts,
routed_scaling_factor, tileN, routing_method_type, do_finalize, mDtypeElt, *runner, configToRun,
topk_weights, topk_ids, output, prerouted, fc1_only);
topk_weights, topk_ids, output, prerouted, fc1_only, tables_max_ctas);
}

std::vector<int32_t> const mSupportedTileN;
Expand Down Expand Up @@ -1038,6 +1120,9 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m)
.def("run_moe_routing_tables", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runRoutingTables)
.def("run_moe_prerouted", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runPrerouted)
.def("run_moe_fc1", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runFc1)
.def("run_moe_routing_tables_fc1", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runRoutingTablesFc1)
.def("run_moe_bounded", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runBounded)
.def("run_moe_bounded_fc1", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runBoundedFc1)
.def("run_moe_prerouted_fc1", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::runPreroutedFc1)
.def("get_gemm2_kernel_name", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::getGemm2KernelName)
.def("get_gemm2_kernel_options", &tensorrt_llm::torch_ext::FP4BlockScaleMoeRunner::getGemm2KernelOptions)
Expand Down
13 changes: 10 additions & 3 deletions tensorrt_llm/_torch/custom_ops/router_logit_partials.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@

import torch

from tensorrt_llm._utils import is_blackwell_datacenter

# Token counts served by partials (up to the hi GEN batch, 320 requests x 4 rows); the router GEMM keeps its
# other paths outside them.
MIN_ROWS = 65
# Fewest rows for callers that opt in (``small_m``) on SM100/SM103: below them the small-M
# warp-MMA router GEMM is faster.
SMALL_M_MIN_ROWS = 12
MAX_ROWS = 1280


Expand Down Expand Up @@ -53,11 +58,13 @@ def num_splits(num_tokens: int, num_experts: int) -> int:
return 8 if num_logits <= 256 * 384 else 4 if num_logits <= 768 * 384 else 2


def applies(x: torch.Tensor, weight: torch.Tensor) -> bool:
"""Whether ``router_logit_partials`` serves ``x @ weight.T``."""
def applies(x: torch.Tensor, weight: torch.Tensor, small_m: bool = False) -> bool:
"""Whether ``router_logit_partials`` serves ``x @ weight.T``; with ``small_m`` from
``SMALL_M_MIN_ROWS`` rows on SM100/SM103."""
num_tokens, hidden = x.shape
min_rows = SMALL_M_MIN_ROWS if small_m and is_blackwell_datacenter() else MIN_ROWS
return (
MIN_ROWS <= num_tokens <= MAX_ROWS
min_rows <= num_tokens <= MAX_ROWS
and x.is_cuda
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
Expand Down
Loading
Loading