Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions tensorrt_llm/_torch/_experimental/modeling_v2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,9 @@ example that uses no Modeling V1 model code.
until it is validated, decorate its class with `@disable_v2_by_default`
(from `tensorrt_llm._torch._experimental.modeling_v2`). Deployments then use
it only when they set `modeling_v2` to `auto` or `require` explicitly.
A target with MoE layers also declares the MoE tactics it honors. See
[Routing](#routing).
A target with MoE layers also declares the MoE tactics it honors, and a
target that implements expert load balancing lists its modes in
`EPLB_MODES`. See [Routing](#routing).
4. Use catalog operations where they fit. Add new operations as `OpWrapper`s
with tests. See [Writing operations](#writing-operations).
5. For C++/CUDA code, follow the [native code guide](../../../../cpp/tensorrt_llm/modeling_v2/README.md).
Expand Down Expand Up @@ -209,6 +210,17 @@ Check in `route` every requirement the target has, not only in the target's
constructor. A configuration that `route` declines falls back to Modeling V1
under `auto`; one that the constructor rejects with an assertion fails.

Expert load balancing (EPLB) is checked for every target. When a deployment
runs EPLB (`moe_config.load_balancer` is set, the architecture is in
`moe_model_arch_list` and the experts are split over more than one EP rank),
the loader creates the load balancer whichever class it builds, but only a
target that registers its MoE layers with it balances anything. After `route`
returns a target, the resolver compares the deployment's EPLB mode (`static`,
`online` or `per_layer_dynamic`) with the target class's `EPLB_MODES` and
treats a target that does not list it as `NoTarget`: `auto` keeps Modeling V1
for that deployment and `require` raises with the reason. `EPLB_MODES` is empty
unless the target sets it.

`route` reads no MoE tactic (`moe_config.tactic`, `speculative_config.moe_tactic`),
so a target with MoE layers declares the MoE implementation ids it runs.
`moe_impls_honored` lists the ids a `moe_config.tactic` may resolve to for the
Expand Down
41 changes: 37 additions & 4 deletions tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,15 +306,47 @@ def _refuse_unhonored_moe_tactic(
)


def _check_eplb_mode(target: type, config: "ModelConfig") -> None:
"""Raise ``NoTarget`` unless ``target`` implements the EPLB mode this deployment runs.

The loader creates the MoE load balancer around whichever class it builds
(``maybe_create_moe_load_balancer``). A target takes part only by
registering its MoE layers with it, so a target that does not would run an
EPLB deployment on fixed expert windows: no replicas under static EPLB and
no layer to update under online EPLB. A target lists the modes it
implements in the class attribute ``EPLB_MODES``; without it, none.
"""
from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import (
MOE_LOAD_BALANCER_MODES,
moe_load_balancer_mode,
)

implemented = frozenset(getattr(target, "EPLB_MODES", ()))
unknown = implemented - MOE_LOAD_BALANCER_MODES
if unknown:
raise ValueError(
f"{target.__qualname__}.EPLB_MODES names {sorted(unknown)}; the EPLB modes are "
f"{sorted(MOE_LOAD_BALANCER_MODES)}"
)
eplb = moe_load_balancer_mode(config, config.mapping)
if eplb is not None and eplb not in implemented:
raise NoTarget(
f"{eplb} EPLB (moe_config.load_balancer at moe_ep_size={config.mapping.moe_ep_size}); "
f"{target.__name__} implements {', '.join(sorted(implemented)) or 'no EPLB mode'}"
)


def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]:
"""Decide which modeling_v2 target, if any, this deployment builds.

Returns the target class, or None for the built-in implementation:
when ``llm_args.modeling_v2`` is ``off``, when no ``route`` is registered
for the architecture, or when that ``route`` raises ``NoTarget``.
In ``require`` every one of those but ``off`` raises ``ValueError`` instead.
It also returns None for a ``@disable_v2_by_default`` target when
``modeling_v2`` was left at its default.
for the architecture, when that ``route`` raises ``NoTarget``, or when
the deployment runs EPLB in a mode the target does not list in
``EPLB_MODES``. In ``require`` every one of those but ``off`` raises
``ValueError`` instead. It also returns None for a
``@disable_v2_by_default`` target when ``modeling_v2`` was left at its
default.

"""
mode = ModelingV2Mode.of(llm_args)
Expand All @@ -333,6 +365,7 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti
raise NoTarget(f"no route for this architecture; routed architectures: {known}")
target = route(config, llm_args, _device_compute_capability())
if target is not None:
_check_eplb_mode(target, config)
_refuse_unhonored_moe_tactics(config, target, arch)
except NoTarget as why:
if mode is ModelingV2Mode.REQUIRE:
Expand Down
42 changes: 40 additions & 2 deletions tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1125,6 +1125,45 @@ def __repr__(self):
'Qwen3_5MoeForConditionalGeneration',
]

#: The EPLB modes ``moe_load_balancer_mode`` reports.
MOE_LOAD_BALANCER_MODES = frozenset({"static", "online", "per_layer_dynamic"})


def moe_load_balancer_mode(model_config,
mapping: Optional[Mapping]) -> Optional[str]:
"""Return the EPLB mode the engine runs this model with, or None.

``maybe_create_moe_load_balancer`` creates a balancer exactly when this
returns a mode, so code that decides before the model is built, such as
the modeling_v2 resolver, agrees with the loader on whether EPLB runs.

Args:
model_config: The model's ``ModelConfig``. Its ``moe_load_balancer``
and the first of ``pretrained_config.architectures`` are read.
mapping: The mapping the model is built with.

Returns:
``"static"`` for standard EPLB that keeps its initial placement
(``layer_updates_per_iter == 0``), ``"online"`` for standard EPLB
that updates layers, ``"per_layer_dynamic"`` for per-layer dynamic
EPLB, and None when no balancer is created: no ``moe_load_balancer``
config, an architecture outside ``moe_model_arch_list``, a single EP
rank, or smart routing (``moe_cluster_size > 1``).
"""
config = model_config.moe_load_balancer
if config is None or mapping is None:
return None
architectures = getattr(model_config.pretrained_config, "architectures",
None) or [None]
if architectures[0] not in moe_model_arch_list:
return None
if mapping.moe_ep_size <= 1 or mapping.moe_cluster_size > 1:
return None
mode = getattr(config, "mode", "standard")
if mode == "standard":
return "online" if config.layer_updates_per_iter > 0 else "static"
return mode


def maybe_create_moe_load_balancer(
model_config, mapping: Optional[Mapping]
Expand Down Expand Up @@ -1169,8 +1208,7 @@ def maybe_create_moe_load_balancer(
"warmup placement requires pure expert parallelism: "
f"world size {mapping.world_size} != moe_ep_size {ep_size}")

if (in_supported_model_arch and using_ep and not using_smart_router
and config is not None):
if moe_load_balancer_mode(model_config, mapping) is not None:
config.setup(ep_rank=ep_rank, ep_size=ep_size)
if mode == "per_layer_dynamic":
from .per_layer_dynamic_eplb import PerLayerDynamicMoeLoadBalancer
Expand Down
123 changes: 119 additions & 4 deletions tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,14 @@
from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce
from tensorrt_llm._torch.model_config import KIMI_K3_DEFAULT_ROUTED_QUANT_ALGO, ModelConfig
from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM
from tensorrt_llm._torch.moe.fused_moe import moe_load_balancer
from tensorrt_llm._torch.moe.fused_moe.impl_identity import MoEImplId
from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import MOE_LOAD_BALANCER_MODES
from tensorrt_llm.llmapi.llm_args import (
CacheTransceiverConfig,
KvCacheConfig,
MoeLoadBalancerConfig,
MoePerLayerDynamicLoadBalancerConfig,
MTPDecodingConfig,
NGramDecodingConfig,
TorchLlmArgs,
Expand Down Expand Up @@ -241,11 +245,13 @@ class _TargetDisabledByDefault:
"""A target the default mode declines."""


def _claimed_by(monkeypatch, target):
_CLAIMED_ARCH = "ModelingV2TestForCausalLM"


def _claimed_by(monkeypatch, target, **mapping_kwargs):
"""A config whose architecture's ``route`` claims every deployment for ``target``."""
arch = "ModelingV2TestForCausalLM"
monkeypatch.setitem(_ROUTES, arch, lambda *_: target)
return _model_config(PretrainedConfig(architectures=[arch]))
monkeypatch.setitem(_ROUTES, _CLAIMED_ARCH, lambda *_: target)
return _model_config(PretrainedConfig(architectures=[_CLAIMED_ARCH]), **mapping_kwargs)


def test_the_default_mode_builds_a_claiming_target(monkeypatch):
Expand All @@ -265,6 +271,89 @@ def test_a_set_mode_builds_a_target_disabled_by_default(monkeypatch, mode):
assert modeling_v2_resolve(config, _llm_args(mode)) is _TargetDisabledByDefault


# --- Expert load balancing: a target runs only the EPLB modes it lists -----------------------


class _TargetWithStaticEplb:
"""A target that implements static EPLB and no other mode."""

EPLB_MODES = frozenset({"static"})


def _eplb_config(eplb_mode, num_slots=288):
"""A ``moe_config.load_balancer`` that runs ``eplb_mode``."""
if eplb_mode == "per_layer_dynamic":
return MoePerLayerDynamicLoadBalancerConfig(num_slots=num_slots)
updates = 2 if eplb_mode == "online" else 0
return MoeLoadBalancerConfig(num_slots=num_slots, layer_updates_per_iter=updates)


def _balanced(config, eplb_mode, num_slots=288):
"""``config`` deployed with ``eplb_mode``, as the loader hands it to routing."""
return replace(config, moe_load_balancer=_eplb_config(eplb_mode, num_slots))


def _balance_the_claimed_arch(monkeypatch):
"""Have the engine balance ``_CLAIMED_ARCH``, as it balances the architectures in moe_model_arch_list."""
monkeypatch.setattr(
moe_load_balancer,
"moe_model_arch_list",
[*moe_load_balancer.moe_model_arch_list, _CLAIMED_ARCH],
)


@pytest.mark.parametrize("eplb_mode", sorted(MOE_LOAD_BALANCER_MODES))
def test_eplb_declines_a_target_that_lists_no_mode(monkeypatch, eplb_mode):
"""The loader creates the load balancer around whatever class it builds. A target that registers no
layer with it would serve the deployment on fixed expert windows, so ``auto`` keeps the built-in
model and ``require`` says why."""
_balance_the_claimed_arch(monkeypatch)
config = _balanced(_claimed_by(monkeypatch, _TargetUsedByDefault, **_DEP4), eplb_mode)
assert modeling_v2_resolve(config, _llm_args("auto")) is None
with pytest.raises(ValueError, match=f"{eplb_mode} EPLB .* implements no EPLB mode"):
modeling_v2_resolve(config, _llm_args("require"))


@pytest.mark.parametrize("eplb_mode", sorted(MOE_LOAD_BALANCER_MODES))
def test_eplb_builds_a_target_for_the_modes_it_lists(monkeypatch, eplb_mode):
_balance_the_claimed_arch(monkeypatch)
config = _balanced(_claimed_by(monkeypatch, _TargetWithStaticEplb, **_DEP4), eplb_mode)
expected = _TargetWithStaticEplb if eplb_mode == "static" else None
assert modeling_v2_resolve(config, _llm_args("auto")) is expected


@pytest.mark.parametrize(
"arch_balanced, mapping_kwargs, eplb_mode",
[
pytest.param(True, _DEP4, None, id="no-load-balancer-config"),
pytest.param(True, {}, "static", id="one-ep-rank"),
pytest.param(False, _DEP4, "static", id="architecture-not-balanced"),
],
)
def test_eplb_config_without_a_balancer_changes_nothing(
monkeypatch, arch_balanced, mapping_kwargs, eplb_mode
):
"""Where the engine creates no balancer, the load-balancer config is inert, and a target that lists no
EPLB mode is built as before."""
if arch_balanced:
_balance_the_claimed_arch(monkeypatch)
config = _claimed_by(monkeypatch, _TargetUsedByDefault, **mapping_kwargs)
if eplb_mode is not None:
config = _balanced(config, eplb_mode)
assert modeling_v2_resolve(config, _llm_args("require")) is _TargetUsedByDefault


def test_an_unknown_eplb_mode_name_is_an_error(monkeypatch):
"""A misspelled mode never matches, so the target would decline every EPLB deployment without a word."""

class _TargetWithMisspelledMode:
EPLB_MODES = frozenset({"Static"})

config = _claimed_by(monkeypatch, _TargetWithMisspelledMode)
with pytest.raises(ValueError, match=r"EPLB_MODES names \['Static'\]"):
modeling_v2_resolve(config, _llm_args("auto"))


@pytest.mark.parametrize(
"pretrained_config, expected",
[(_gpt_oss_config, _GPT_OSS_DEFAULT_TARGET), (_r1_config, _R1_DEFAULT_TARGET)],
Expand Down Expand Up @@ -1832,6 +1921,32 @@ def test_glm52_without_an_fp8_kv_cache_does_not_match():
modeling_v2_resolve(config, _llm_args("require"))


# --- EPLB on the targets that do not implement it -------------------------------------------


@pytest.mark.parametrize("eplb_mode", ["static", "online"])
@pytest.mark.parametrize(
"pretrained_config, num_slots",
[pytest.param(_r1_config, 288, id="r1"), pytest.param(_gpt_oss_config, 160, id="gpt_oss")],
)
def test_eplb_at_dep4_keeps_the_builtin_r1_and_gpt_oss(pretrained_config, num_slots, eplb_mode):
"""The built-in DeepSeek-R1 and gpt-oss models balance their experts; the default targets register no
MoE layer with the balancer, so an EPLB deployment that sets modeling_v2 keeps the built-in model."""
config = _balanced(_model_config(pretrained_config(), **_DEP4), eplb_mode, num_slots)
assert modeling_v2_resolve(config, _llm_args("auto")) is None
with pytest.raises(ValueError, match=f"{eplb_mode} EPLB .* implements no EPLB mode"):
modeling_v2_resolve(config, _llm_args("require"))


@pytest.mark.usefixtures("_glm52_sm")
def test_glm52_tep4_with_eplb_keeps_the_builtin_model():
"""The tep4 target is built by default; it must not serve an EPLB deployment on fixed expert windows."""
config = _balanced(_model_config(_glm52_config(), **_TEP4), "static")
assert modeling_v2_resolve(config, TorchLlmArgs(model=_DUMMY_MODEL)) is None
with pytest.raises(ValueError, match="static EPLB .* implements no EPLB mode"):
modeling_v2_resolve(config, _llm_args("require"))


# --- GLM-5.2 on sm107: MoE tactics ------------------------------------------------

_GLM52_SM107_TARGET = (
Expand Down
Loading
Loading