diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 181b58a5d3c6..a5c1dfa0d785 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -158,8 +158,9 @@ example that uses no Modeling V1 model code. until it is validated, decorate its class with `@disable_v2_by_default` (from `tensorrt_llm._torch._experimental.modeling_v2`). Deployments then use it only when they set `modeling_v2` to `auto` or `require` explicitly. - A target with MoE layers also declares the MoE tactics it honors. See - [Routing](#routing). + A target with MoE layers also declares the MoE tactics it honors, and a + target that implements expert load balancing lists its modes in + `EPLB_MODES`. See [Routing](#routing). 4. Use catalog operations where they fit. Add new operations as `OpWrapper`s with tests. See [Writing operations](#writing-operations). 5. For C++/CUDA code, follow the [native code guide](../../../../cpp/tensorrt_llm/modeling_v2/README.md). @@ -209,6 +210,17 @@ Check in `route` every requirement the target has, not only in the target's constructor. A configuration that `route` declines falls back to Modeling V1 under `auto`; one that the constructor rejects with an assertion fails. +Expert load balancing (EPLB) is checked for every target. When a deployment +runs EPLB (`moe_config.load_balancer` is set, the architecture is in +`moe_model_arch_list` and the experts are split over more than one EP rank), +the loader creates the load balancer whichever class it builds, but only a +target that registers its MoE layers with it balances anything. After `route` +returns a target, the resolver compares the deployment's EPLB mode (`static`, +`online` or `per_layer_dynamic`) with the target class's `EPLB_MODES` and +treats a target that does not list it as `NoTarget`: `auto` keeps Modeling V1 +for that deployment and `require` raises with the reason. `EPLB_MODES` is empty +unless the target sets it. + `route` reads no MoE tactic (`moe_config.tactic`, `speculative_config.moe_tactic`), so a target with MoE layers declares the MoE implementation ids it runs. `moe_impls_honored` lists the ids a `moe_config.tactic` may resolve to for the diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py index 56570da39a4e..b3630fb2be93 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py @@ -306,15 +306,47 @@ def _refuse_unhonored_moe_tactic( ) +def _check_eplb_mode(target: type, config: "ModelConfig") -> None: + """Raise ``NoTarget`` unless ``target`` implements the EPLB mode this deployment runs. + + The loader creates the MoE load balancer around whichever class it builds + (``maybe_create_moe_load_balancer``). A target takes part only by + registering its MoE layers with it, so a target that does not would run an + EPLB deployment on fixed expert windows: no replicas under static EPLB and + no layer to update under online EPLB. A target lists the modes it + implements in the class attribute ``EPLB_MODES``; without it, none. + """ + from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import ( + MOE_LOAD_BALANCER_MODES, + moe_load_balancer_mode, + ) + + implemented = frozenset(getattr(target, "EPLB_MODES", ())) + unknown = implemented - MOE_LOAD_BALANCER_MODES + if unknown: + raise ValueError( + f"{target.__qualname__}.EPLB_MODES names {sorted(unknown)}; the EPLB modes are " + f"{sorted(MOE_LOAD_BALANCER_MODES)}" + ) + eplb = moe_load_balancer_mode(config, config.mapping) + if eplb is not None and eplb not in implemented: + raise NoTarget( + f"{eplb} EPLB (moe_config.load_balancer at moe_ep_size={config.mapping.moe_ep_size}); " + f"{target.__name__} implements {', '.join(sorted(implemented)) or 'no EPLB mode'}" + ) + + def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: """Decide which modeling_v2 target, if any, this deployment builds. Returns the target class, or None for the built-in implementation: when ``llm_args.modeling_v2`` is ``off``, when no ``route`` is registered - for the architecture, or when that ``route`` raises ``NoTarget``. - In ``require`` every one of those but ``off`` raises ``ValueError`` instead. - It also returns None for a ``@disable_v2_by_default`` target when - ``modeling_v2`` was left at its default. + for the architecture, when that ``route`` raises ``NoTarget``, or when + the deployment runs EPLB in a mode the target does not list in + ``EPLB_MODES``. In ``require`` every one of those but ``off`` raises + ``ValueError`` instead. It also returns None for a + ``@disable_v2_by_default`` target when ``modeling_v2`` was left at its + default. """ mode = ModelingV2Mode.of(llm_args) @@ -333,6 +365,7 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti raise NoTarget(f"no route for this architecture; routed architectures: {known}") target = route(config, llm_args, _device_compute_capability()) if target is not None: + _check_eplb_mode(target, config) _refuse_unhonored_moe_tactics(config, target, arch) except NoTarget as why: if mode is ModelingV2Mode.REQUIRE: diff --git a/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py b/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py index 483cdd00e307..62d960ebc109 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py +++ b/tensorrt_llm/_torch/moe/fused_moe/moe_load_balancer.py @@ -1125,6 +1125,45 @@ def __repr__(self): 'Qwen3_5MoeForConditionalGeneration', ] +#: The EPLB modes ``moe_load_balancer_mode`` reports. +MOE_LOAD_BALANCER_MODES = frozenset({"static", "online", "per_layer_dynamic"}) + + +def moe_load_balancer_mode(model_config, + mapping: Optional[Mapping]) -> Optional[str]: + """Return the EPLB mode the engine runs this model with, or None. + + ``maybe_create_moe_load_balancer`` creates a balancer exactly when this + returns a mode, so code that decides before the model is built, such as + the modeling_v2 resolver, agrees with the loader on whether EPLB runs. + + Args: + model_config: The model's ``ModelConfig``. Its ``moe_load_balancer`` + and the first of ``pretrained_config.architectures`` are read. + mapping: The mapping the model is built with. + + Returns: + ``"static"`` for standard EPLB that keeps its initial placement + (``layer_updates_per_iter == 0``), ``"online"`` for standard EPLB + that updates layers, ``"per_layer_dynamic"`` for per-layer dynamic + EPLB, and None when no balancer is created: no ``moe_load_balancer`` + config, an architecture outside ``moe_model_arch_list``, a single EP + rank, or smart routing (``moe_cluster_size > 1``). + """ + config = model_config.moe_load_balancer + if config is None or mapping is None: + return None + architectures = getattr(model_config.pretrained_config, "architectures", + None) or [None] + if architectures[0] not in moe_model_arch_list: + return None + if mapping.moe_ep_size <= 1 or mapping.moe_cluster_size > 1: + return None + mode = getattr(config, "mode", "standard") + if mode == "standard": + return "online" if config.layer_updates_per_iter > 0 else "static" + return mode + def maybe_create_moe_load_balancer( model_config, mapping: Optional[Mapping] @@ -1169,8 +1208,7 @@ def maybe_create_moe_load_balancer( "warmup placement requires pure expert parallelism: " f"world size {mapping.world_size} != moe_ep_size {ep_size}") - if (in_supported_model_arch and using_ep and not using_smart_router - and config is not None): + if moe_load_balancer_mode(model_config, mapping) is not None: config.setup(ep_rank=ep_rank, ep_size=ep_size) if mode == "per_layer_dynamic": from .per_layer_dynamic_eplb import PerLayerDynamicMoeLoadBalancer diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py index 26fd063dfb92..55a4c6ea24f2 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_routing.py @@ -52,10 +52,14 @@ from tensorrt_llm._torch.distributed.ops import MNNVLAllReduce from tensorrt_llm._torch.model_config import KIMI_K3_DEFAULT_ROUTED_QUANT_ALGO, ModelConfig from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM +from tensorrt_llm._torch.moe.fused_moe import moe_load_balancer from tensorrt_llm._torch.moe.fused_moe.impl_identity import MoEImplId +from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import MOE_LOAD_BALANCER_MODES from tensorrt_llm.llmapi.llm_args import ( CacheTransceiverConfig, KvCacheConfig, + MoeLoadBalancerConfig, + MoePerLayerDynamicLoadBalancerConfig, MTPDecodingConfig, NGramDecodingConfig, TorchLlmArgs, @@ -241,11 +245,13 @@ class _TargetDisabledByDefault: """A target the default mode declines.""" -def _claimed_by(monkeypatch, target): +_CLAIMED_ARCH = "ModelingV2TestForCausalLM" + + +def _claimed_by(monkeypatch, target, **mapping_kwargs): """A config whose architecture's ``route`` claims every deployment for ``target``.""" - arch = "ModelingV2TestForCausalLM" - monkeypatch.setitem(_ROUTES, arch, lambda *_: target) - return _model_config(PretrainedConfig(architectures=[arch])) + monkeypatch.setitem(_ROUTES, _CLAIMED_ARCH, lambda *_: target) + return _model_config(PretrainedConfig(architectures=[_CLAIMED_ARCH]), **mapping_kwargs) def test_the_default_mode_builds_a_claiming_target(monkeypatch): @@ -265,6 +271,89 @@ def test_a_set_mode_builds_a_target_disabled_by_default(monkeypatch, mode): assert modeling_v2_resolve(config, _llm_args(mode)) is _TargetDisabledByDefault +# --- Expert load balancing: a target runs only the EPLB modes it lists ----------------------- + + +class _TargetWithStaticEplb: + """A target that implements static EPLB and no other mode.""" + + EPLB_MODES = frozenset({"static"}) + + +def _eplb_config(eplb_mode, num_slots=288): + """A ``moe_config.load_balancer`` that runs ``eplb_mode``.""" + if eplb_mode == "per_layer_dynamic": + return MoePerLayerDynamicLoadBalancerConfig(num_slots=num_slots) + updates = 2 if eplb_mode == "online" else 0 + return MoeLoadBalancerConfig(num_slots=num_slots, layer_updates_per_iter=updates) + + +def _balanced(config, eplb_mode, num_slots=288): + """``config`` deployed with ``eplb_mode``, as the loader hands it to routing.""" + return replace(config, moe_load_balancer=_eplb_config(eplb_mode, num_slots)) + + +def _balance_the_claimed_arch(monkeypatch): + """Have the engine balance ``_CLAIMED_ARCH``, as it balances the architectures in moe_model_arch_list.""" + monkeypatch.setattr( + moe_load_balancer, + "moe_model_arch_list", + [*moe_load_balancer.moe_model_arch_list, _CLAIMED_ARCH], + ) + + +@pytest.mark.parametrize("eplb_mode", sorted(MOE_LOAD_BALANCER_MODES)) +def test_eplb_declines_a_target_that_lists_no_mode(monkeypatch, eplb_mode): + """The loader creates the load balancer around whatever class it builds. A target that registers no + layer with it would serve the deployment on fixed expert windows, so ``auto`` keeps the built-in + model and ``require`` says why.""" + _balance_the_claimed_arch(monkeypatch) + config = _balanced(_claimed_by(monkeypatch, _TargetUsedByDefault, **_DEP4), eplb_mode) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{eplb_mode} EPLB .* implements no EPLB mode"): + modeling_v2_resolve(config, _llm_args("require")) + + +@pytest.mark.parametrize("eplb_mode", sorted(MOE_LOAD_BALANCER_MODES)) +def test_eplb_builds_a_target_for_the_modes_it_lists(monkeypatch, eplb_mode): + _balance_the_claimed_arch(monkeypatch) + config = _balanced(_claimed_by(monkeypatch, _TargetWithStaticEplb, **_DEP4), eplb_mode) + expected = _TargetWithStaticEplb if eplb_mode == "static" else None + assert modeling_v2_resolve(config, _llm_args("auto")) is expected + + +@pytest.mark.parametrize( + "arch_balanced, mapping_kwargs, eplb_mode", + [ + pytest.param(True, _DEP4, None, id="no-load-balancer-config"), + pytest.param(True, {}, "static", id="one-ep-rank"), + pytest.param(False, _DEP4, "static", id="architecture-not-balanced"), + ], +) +def test_eplb_config_without_a_balancer_changes_nothing( + monkeypatch, arch_balanced, mapping_kwargs, eplb_mode +): + """Where the engine creates no balancer, the load-balancer config is inert, and a target that lists no + EPLB mode is built as before.""" + if arch_balanced: + _balance_the_claimed_arch(monkeypatch) + config = _claimed_by(monkeypatch, _TargetUsedByDefault, **mapping_kwargs) + if eplb_mode is not None: + config = _balanced(config, eplb_mode) + assert modeling_v2_resolve(config, _llm_args("require")) is _TargetUsedByDefault + + +def test_an_unknown_eplb_mode_name_is_an_error(monkeypatch): + """A misspelled mode never matches, so the target would decline every EPLB deployment without a word.""" + + class _TargetWithMisspelledMode: + EPLB_MODES = frozenset({"Static"}) + + config = _claimed_by(monkeypatch, _TargetWithMisspelledMode) + with pytest.raises(ValueError, match=r"EPLB_MODES names \['Static'\]"): + modeling_v2_resolve(config, _llm_args("auto")) + + @pytest.mark.parametrize( "pretrained_config, expected", [(_gpt_oss_config, _GPT_OSS_DEFAULT_TARGET), (_r1_config, _R1_DEFAULT_TARGET)], @@ -1832,6 +1921,32 @@ def test_glm52_without_an_fp8_kv_cache_does_not_match(): modeling_v2_resolve(config, _llm_args("require")) +# --- EPLB on the targets that do not implement it ------------------------------------------- + + +@pytest.mark.parametrize("eplb_mode", ["static", "online"]) +@pytest.mark.parametrize( + "pretrained_config, num_slots", + [pytest.param(_r1_config, 288, id="r1"), pytest.param(_gpt_oss_config, 160, id="gpt_oss")], +) +def test_eplb_at_dep4_keeps_the_builtin_r1_and_gpt_oss(pretrained_config, num_slots, eplb_mode): + """The built-in DeepSeek-R1 and gpt-oss models balance their experts; the default targets register no + MoE layer with the balancer, so an EPLB deployment that sets modeling_v2 keeps the built-in model.""" + config = _balanced(_model_config(pretrained_config(), **_DEP4), eplb_mode, num_slots) + assert modeling_v2_resolve(config, _llm_args("auto")) is None + with pytest.raises(ValueError, match=f"{eplb_mode} EPLB .* implements no EPLB mode"): + modeling_v2_resolve(config, _llm_args("require")) + + +@pytest.mark.usefixtures("_glm52_sm") +def test_glm52_tep4_with_eplb_keeps_the_builtin_model(): + """The tep4 target is built by default; it must not serve an EPLB deployment on fixed expert windows.""" + config = _balanced(_model_config(_glm52_config(), **_TEP4), "static") + assert modeling_v2_resolve(config, TorchLlmArgs(model=_DUMMY_MODEL)) is None + with pytest.raises(ValueError, match="static EPLB .* implements no EPLB mode"): + modeling_v2_resolve(config, _llm_args("require")) + + # --- GLM-5.2 on sm107: MoE tactics ------------------------------------------------ _GLM52_SM107_TARGET = ( diff --git a/tests/unittest/_torch/moe/test_moe_load_balancer.py b/tests/unittest/_torch/moe/test_moe_load_balancer.py index 79185e3c2f2f..0c14eea80e24 100644 --- a/tests/unittest/_torch/moe/test_moe_load_balancer.py +++ b/tests/unittest/_torch/moe/test_moe_load_balancer.py @@ -15,8 +15,9 @@ from tensorrt_llm._torch.moe.fused_moe.moe_load_balancer import ( MoeLoadBalancer, MoeLoadBalancerIterContext, SingleLayerMoeLoadBalancer, get_moe_load_balancer, maybe_create_moe_load_balancer, - moe_load_balancer_add_single_layer) -from tensorrt_llm.llmapi.llm_args import MoeLoadBalancerConfig + moe_load_balancer_add_single_layer, moe_load_balancer_mode) +from tensorrt_llm.llmapi.llm_args import (MoeLoadBalancerConfig, + MoePerLayerDynamicLoadBalancerConfig) from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -585,5 +586,91 @@ def create_decoder(decoder_config: ModelConfig) -> torch.nn.Module: assert get_moe_load_balancer() is None +_EP4 = dict(world_size=4, rank=1, tp_size=4, moe_ep_size=4) + +# (architecture, mapping kwargs, load-balancer config, the mode the engine +# runs). The standard-mode and no-balancer cases, which +# maybe_create_moe_load_balancer can build here with the runtime mocked. +_STANDARD_MODE_CASES = [ + pytest.param("DeepseekV3ForCausalLM", + _EP4, + MoeLoadBalancerConfig(num_slots=288), + "static", + id="static"), + pytest.param("DeepseekV3ForCausalLM", + _EP4, + MoeLoadBalancerConfig(num_slots=288, layer_updates_per_iter=2), + "online", + id="online"), + pytest.param("DeepseekV3ForCausalLM", + _EP4, + None, + None, + id="no-load-balancer-config"), + pytest.param("DeepseekV3ForCausalLM", + dict(world_size=4, + rank=1, + tp_size=4, + moe_tp_size=4, + moe_ep_size=1), + MoeLoadBalancerConfig(num_slots=256), + None, + id="one-ep-rank"), + pytest.param("LlamaForCausalLM", + _EP4, + MoeLoadBalancerConfig(num_slots=288), + None, + id="architecture-not-balanced"), +] + + +def _balancer_model_config(arch, mapping, load_balancer): + pretrained_config = PretrainedConfig() + pretrained_config.architectures = [arch] + return ModelConfig(pretrained_config=pretrained_config, + mapping=mapping, + moe_load_balancer=load_balancer) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "arch, mapping_kwargs, load_balancer, expected", _STANDARD_MODE_CASES + [ + pytest.param("DeepseekV3ForCausalLM", + _EP4, + MoePerLayerDynamicLoadBalancerConfig(num_slots=288), + "per_layer_dynamic", + id="per-layer-dynamic"), + ]) +def test_moe_load_balancer_mode(arch, mapping_kwargs, load_balancer, + expected) -> None: + mapping = Mapping(**mapping_kwargs) + model_config = _balancer_model_config(arch, mapping, load_balancer) + + assert moe_load_balancer_mode(model_config, mapping) == expected + assert moe_load_balancer_mode(model_config, None) is None + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("arch, mapping_kwargs, load_balancer, expected", + _STANDARD_MODE_CASES) +def test_maybe_create_moe_load_balancer_follows_the_mode( + arch, mapping_kwargs, load_balancer, expected, + mock_runtime: MagicMock) -> None: + """The loader creates a balancer exactly where the mode says EPLB runs.""" + mapping = Mapping(**mapping_kwargs) + model_config = _balancer_model_config(arch, mapping, load_balancer) + + with maybe_create_moe_load_balancer(model_config, mapping) as balancer: + if expected is None: + assert balancer is None + else: + try: + assert isinstance(balancer, MoeLoadBalancer) + assert balancer.is_static_routing() == (expected == "static") + finally: + balancer.shutdown() + assert get_moe_load_balancer() is None + + if __name__ == '__main__': unittest.main()