From f38408d4e89887f3da293510e20c6f5fcc41bc98 Mon Sep 17 00:00:00 2001 From: Jeremy Berchtold Date: Wed, 2 Sep 2026 14:50:23 -0700 Subject: [PATCH 01/15] [JAX] Support compound EP axes in MoEBlock Signed-off-by: Jeremy Berchtold --- tests/jax/test_multi_process_ep.py | 31 +++- transformer_engine/jax/cpp_extensions/ep.py | 149 +++++++++++++++----- transformer_engine/jax/ep.py | 108 ++++++++------ transformer_engine/jax/flax/moe.py | 4 + transformer_engine/jax/moe.py | 53 ++++--- 5 files changed, 251 insertions(+), 94 deletions(-) diff --git a/tests/jax/test_multi_process_ep.py b/tests/jax/test_multi_process_ep.py index 3882281237..a0e7a6a5a1 100644 --- a/tests/jax/test_multi_process_ep.py +++ b/tests/jax/test_multi_process_ep.py @@ -53,6 +53,8 @@ ep_dispatch_fwd, ep_combine_fwd, get_ep_config, + _leading_axis_ok, + _ep_spec_ok, is_ep_borrowed_comm_built, use_nccl_comm_from_xla, ) @@ -62,7 +64,6 @@ is_xla_ffi_collectives_supported, ) - # ── Test config ───────────────────────────────────────────────────────────── # NCCL EP requires NUM_LOCAL_EXPERTS*ep % 4 == 0 (TMA alignment in # device/hybridep_adapter.cu:511). With NUM_LOCAL_EXPERTS=2, ep must be even. @@ -1418,6 +1419,34 @@ def test_ep_tp_splits_domains(self): self.assertEqual(domains, {0: [0, 2, 4, 6], 1: [1, 3, 5, 7]}) + def test_compound_ep_axis(self): + if not is_devices_enough(4): + self.skipTest("requires 4 devices") + mesh = Mesh(np.asarray(jax.devices()[:4]).reshape(2, 2), ("expert", "tensor")) + order = {int(device.id): i for i, device in enumerate(mesh.devices.reshape(-1))} + + domains = {} + for rank in range(4): + root, col, num_domains = _ep_domain_for_rank( + mesh, + ("expert", "tensor"), + rank, + device_to_rank=lambda device: order[int(device.id)], + ) + self.assertEqual(num_domains, 1) + domains.setdefault(root, {})[col] = rank + self.assertEqual([domains[0][i] for i in range(4)], [0, 1, 2, 3]) + + with mesh: + ep_axes = ("expert", "tensor") + ok, ep_axis, outer_axes = _leading_axis_ok( + PartitionSpec(ep_axes, None, None), ep_axes=ep_axes + ) + self.assertTrue(ok) + self.assertEqual(ep_axis, ("expert", "tensor")) + self.assertEqual(outer_axes, ()) + self.assertTrue(_ep_spec_ok(PartitionSpec(ep_axes, None, None), 2, ep_axes=ep_axes)) + def test_ep_flattened_replica_groups_multi_domain(self): """Verify flattened replica IDs are grouped by DP coordinate.""" if not is_devices_enough(4): diff --git a/transformer_engine/jax/cpp_extensions/ep.py b/transformer_engine/jax/cpp_extensions/ep.py index e68a85e2d8..9c7a036473 100644 --- a/transformer_engine/jax/cpp_extensions/ep.py +++ b/transformer_engine/jax/cpp_extensions/ep.py @@ -18,7 +18,7 @@ import os import warnings from dataclasses import dataclass -from typing import Union +from typing import Tuple, Union import jax import jax.numpy as jnp @@ -237,6 +237,34 @@ def empty_token_scale(tokens): return jnp.zeros(tokens.shape[:-1] + (0,), jnp.float8_e8m0fnu) +def _normalize_ep_axes(ep_axis: Union[str, Tuple[str, ...]]) -> Tuple[str, ...]: + """Normalize a physical EP mesh axis or axis tuple.""" + axes = (ep_axis,) if isinstance(ep_axis, str) else ep_axis + if not isinstance(axes, tuple): + raise ValueError(f"ep_axis must be a mesh axis name or tuple of names, got {ep_axis!r}") + if not axes or any(not isinstance(axis, str) or not axis for axis in axes): + raise ValueError( + f"ep_axis must be a mesh axis name or non-empty tuple of names, got {ep_axis!r}" + ) + if len(set(axes)) != len(axes): + raise ValueError(f"ep_axis must not contain duplicate mesh axes, got {ep_axis!r}") + return axes + + +def _ep_axis_entry(ep_axes: Tuple[str, ...]): + """Return the PartitionSpec entry for normalized EP axes.""" + return ep_axes[0] if len(ep_axes) == 1 else ep_axes + + +def _resolve_ep_axes(ep_axes=None) -> Tuple[str, ...]: + """Resolve explicit EP axes, falling back to MeshResource for compatibility.""" + if ep_axes is None: + ep_axes = global_mesh_resource().ep_resource + if ep_axes is None: + raise ValueError("EP mesh axes must be passed explicitly or set as ep_resource.") + return _normalize_ep_axes(ep_axes) + + def _fsdp_outer_axes(resource): """FSDP axes of ``resource`` that count as EP outer axes, as a tuple.""" fsdp_resource = resource.fsdp_resource @@ -246,21 +274,27 @@ def _fsdp_outer_axes(resource): return () if fsdp_resource is None else (fsdp_resource,) -def _capture_ep_resource_axes(): +def _capture_ep_resource_axes(ep_axes=None): """Capture physical axes at bind time for asynchronous SPMD partitioning.""" - resource = global_mesh_resource() + try: + resource = global_mesh_resource() + except AssertionError: + if ep_axes is None: + raise + return _ep_axis_entry(_normalize_ep_axes(ep_axes)), () outer_axes = tuple( dict.fromkeys( axis for axis in (resource.dp_resource, *_fsdp_outer_axes(resource)) if axis is not None ) ) - return resource.ep_resource, outer_axes + ep_axes = _resolve_ep_axes(ep_axes) + return _ep_axis_entry(ep_axes), tuple(a for a in outer_axes if a not in ep_axes) -def _leading_axis_ok(spec, resource_axes=None, mesh=None): +def _leading_axis_ok(spec, resource_axes=None, mesh=None, *, ep_axes=None): """Validate replicated trailing dimensions and EP plus optional outer axes.""" ep_axis, outer_axes = ( - resource_axes if resource_axes is not None else _capture_ep_resource_axes() + resource_axes if resource_axes is not None else _capture_ep_resource_axes(ep_axes) ) if len(spec) < 2 or ep_axis is None: return False, ep_axis, outer_axes @@ -270,22 +304,29 @@ def _leading_axis_ok(spec, resource_axes=None, mesh=None): elts = leading if isinstance(leading, tuple) else (leading,) elts = tuple(axis for axis in elts if axis is not None) # JAX canonicalizes size-one mesh axes out of partition specs. - if ep_axis not in elts and (mesh is None or mesh.shape.get(ep_axis) != 1): + if not all( + axis in elts or (mesh is not None and mesh.shape.get(axis) == 1) + for axis in _normalize_ep_axes(ep_axis) + ): return False, ep_axis, outer_axes - allowed = set(outer_axes) | {ep_axis} + allowed = set(outer_axes) | set(_normalize_ep_axes(ep_axis)) return all(a in allowed for a in elts), ep_axis, outer_axes -def _ep_outer_axis(): +def _ep_outer_axis(ep_axes=None): """Legacy single outer axis used by the standalone EP custom VJP wrapper.""" + if ep_axes is not None: + return None resource = global_mesh_resource() - fsdp_axes = _fsdp_outer_axes(resource) + ep_axes = _resolve_ep_axes(ep_axes) + dp_resource = None if resource.dp_resource in ep_axes else resource.dp_resource + fsdp_axes = tuple(axis for axis in _fsdp_outer_axes(resource) if axis not in ep_axes) fsdp_resource = fsdp_axes[0] if len(fsdp_axes) == 1 else fsdp_axes or None - if resource.dp_resource is not None and get_mesh_axis_size(resource.dp_resource) > 1: - return resource.dp_resource + if dp_resource is not None and get_mesh_axis_size(dp_resource) > 1: + return dp_resource if fsdp_resource is not None and get_mesh_axis_size(fsdp_resource) > 1: return fsdp_resource - return resource.dp_resource or fsdp_resource + return dp_resource or fsdp_resource def _ep_leading_dims(is_outer): @@ -302,14 +343,14 @@ def _ep_output_spec(*trailing, resource_axes=None): ep_axis, outer_axes = ( resource_axes if resource_axes is not None else _capture_ep_resource_axes() ) - leading = (*outer_axes, ep_axis) if outer_axes else ep_axis + leading = (*outer_axes, *_normalize_ep_axes(ep_axis)) if outer_axes else ep_axis return PartitionSpec(leading, *trailing) -def _ep_spec_ok(spec, trailing_count, resource_axes=None): +def _ep_spec_ok(spec, trailing_count, resource_axes=None, *, ep_axes=None): """Validate EP-output axes, allowing JAX to drop size-one mesh axes.""" ep_axis, outer_axes = ( - resource_axes if resource_axes is not None else _capture_ep_resource_axes() + resource_axes if resource_axes is not None else _capture_ep_resource_axes(ep_axes) ) if len(spec) != 1 + trailing_count: return False @@ -318,7 +359,7 @@ def _ep_spec_ok(spec, trailing_count, resource_axes=None): leading = spec[0] elts = leading if isinstance(leading, tuple) else (leading,) actual = frozenset(a for a in elts if a is not None) - return actual <= set((*outer_axes, ep_axis)) + return actual <= set((*outer_axes, *_normalize_ep_axes(ep_axis))) # ── ep_prepare ────────────────────────────────────────────────────────────── @@ -1400,7 +1441,7 @@ def shardy_sharding_rule(*args): @_on_collective_stream -def _ep_prepare(cfg: EpLayerConfig, topk_idx): +def _ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): """Exchange routing metadata for ``cfg``; return ``(token_counts, total_recv_tokens, handle_mem)``. ``total_recv_tokens`` is the per-rank pre-drop recv-slot total (includes tokens dropped on overflow).""" @@ -1414,7 +1455,7 @@ def _ep_prepare(cfg: EpLayerConfig, topk_idx): dispatch_output_per_expert_alignment=int(cfg.dispatch_output_per_expert_alignment), topk_idx_dtype=int(cfg.topk_idx_dtype), is_outer=True, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) @@ -1427,6 +1468,7 @@ def _ep_dispatch_fwd( tokens_scale_inv, topk_weights, recv_capacity_per_rank, + ep_axes=None, ): """Scatter tokens and weights to expert ranks; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, handle_mem)``.""" @@ -1441,11 +1483,11 @@ def _ep_dispatch_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) -def ep_prepare(cfg: EpLayerConfig, topk_idx): +def ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): """Deprecated: use ``ep_prepare_and_dispatch_fwd``. Planned for removal in January 2027.""" warnings.warn( "ep_prepare is deprecated and planned to be removed in January 2027. Use" @@ -1453,11 +1495,17 @@ def ep_prepare(cfg: EpLayerConfig, topk_idx): DeprecationWarning, stacklevel=2, ) - return _ep_prepare(cfg, topk_idx) + return _ep_prepare(cfg, topk_idx, ep_axes=ep_axes) def ep_dispatch_fwd( - cfg: EpLayerConfig, handle_mem, topk_idx, tokens, topk_weights, recv_capacity_per_rank + cfg: EpLayerConfig, + handle_mem, + topk_idx, + tokens, + topk_weights, + recv_capacity_per_rank, + ep_axes=None, ): """Deprecated: use ``ep_prepare_and_dispatch_fwd``. Planned for removal in January 2027.""" warnings.warn( @@ -1474,6 +1522,7 @@ def ep_dispatch_fwd( empty_token_scale(tokens), topk_weights, recv_capacity_per_rank, + ep_axes=ep_axes, ) return recv_tokens, recv_topk_weights, handle_mem @@ -1495,7 +1544,13 @@ def _fused_prepare_dispatch_enabled(): @_on_collective_stream def _ep_fused_prepare_and_dispatch_fwd( - cfg: EpLayerConfig, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank + cfg: EpLayerConfig, + topk_idx, + tokens, + tokens_scale_inv, + topk_weights, + recv_capacity_per_rank, + ep_axes=None, ): """Fused prepare + dispatch; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, token_counts, total_recv_tokens, @@ -1510,7 +1565,7 @@ def _ep_fused_prepare_and_dispatch_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) @@ -1520,6 +1575,7 @@ def ep_prepare_and_dispatch_fwd( tokens: Union[jnp.ndarray, ScaledTensor1x], topk_weights, recv_capacity_per_rank, + ep_axes=None, ): """Prepare + dispatch; returns ``(recv_tokens, recv_topk_weights, token_counts, total_recv_tokens, handle_mem)``. @@ -1535,7 +1591,13 @@ def ep_prepare_and_dispatch_fwd( else: tokens_scale_inv = empty_token_scale(tokens) outputs = _ep_prepare_and_dispatch_fwd( - cfg, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank + cfg, + topk_idx, + tokens, + tokens_scale_inv, + topk_weights, + recv_capacity_per_rank, + ep_axes=ep_axes, ) recv_tokens, recv_scale_inv, *rest = outputs if isinstance(tokens_in, ScaledTensor1x): @@ -1552,7 +1614,13 @@ def ep_prepare_and_dispatch_fwd( def _ep_prepare_and_dispatch_fwd( - cfg: EpLayerConfig, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank + cfg: EpLayerConfig, + topk_idx, + tokens, + tokens_scale_inv, + topk_weights, + recv_capacity_per_rank, + ep_axes=None, ): """Prepare + dispatch on raw arrays; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, token_counts, total_recv_tokens, @@ -1562,9 +1630,15 @@ def _ep_prepare_and_dispatch_fwd( """ if _fused_prepare_dispatch_enabled(): return _ep_fused_prepare_and_dispatch_fwd( - cfg, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank + cfg, + topk_idx, + tokens, + tokens_scale_inv, + topk_weights, + recv_capacity_per_rank, + ep_axes=ep_axes, ) - token_counts, total_recv_tokens, handle_mem = _ep_prepare(cfg, topk_idx) + token_counts, total_recv_tokens, handle_mem = _ep_prepare(cfg, topk_idx, ep_axes=ep_axes) recv_tokens, recv_scale_inv, recv_topk_weights, handle_mem = _ep_dispatch_fwd( cfg, handle_mem, @@ -1573,6 +1647,7 @@ def _ep_prepare_and_dispatch_fwd( tokens_scale_inv, topk_weights, recv_capacity_per_rank, + ep_axes=ep_axes, ) return ( recv_tokens, @@ -1586,7 +1661,12 @@ def _ep_prepare_and_dispatch_fwd( @_on_collective_stream def ep_combine_fwd( - cfg: EpLayerConfig, handle_mem, expert_out, num_local_tokens, out_partition_spec=None + cfg: EpLayerConfig, + handle_mem, + expert_out, + num_local_tokens, + out_partition_spec=None, + ep_axes=None, ): """Gather expert outputs back to home ranks. expert_out is pre-weighted.""" out_leading = _normalize_leading_shape(num_local_tokens) @@ -1598,7 +1678,7 @@ def ep_combine_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), out_leading_shape=out_leading, out_partition_spec=out_partition_spec, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) @@ -1610,6 +1690,7 @@ def ep_dispatch_bwd( g_recv_topk_weights, num_local_tokens, out_partition_spec=None, + ep_axes=None, ): """Backward of dispatch; returns (grad_tokens, grad_topk_weights).""" out_leading = _normalize_leading_shape(num_local_tokens) @@ -1622,12 +1703,12 @@ def ep_dispatch_bwd( topk_idx_dtype=int(cfg.topk_idx_dtype), out_leading_shape=out_leading, out_partition_spec=out_partition_spec, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) @_on_collective_stream -def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank): +def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank, ep_axes=None): """Backward of combine; returns grad_expert_out [num_procs, recv_capacity_per_rank, H].""" return EpCombineBwdPrimitive.outer_primitive.bind( handle_mem, @@ -1637,5 +1718,5 @@ def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank) topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(), + resource_axes=_capture_ep_resource_axes(ep_axes), ) diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 4e5c68436a..295aee4372 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -14,7 +14,7 @@ import transformer_engine_jax import transformer_engine.jax.cpp_extensions as tex -from transformer_engine.jax.cpp_extensions.ep import _ep_outer_axis +from transformer_engine.jax.cpp_extensions.ep import _ep_outer_axis, _normalize_ep_axes from transformer_engine.jax.cpp_extensions.misc import jax_dtype_to_te_dtype from transformer_engine.jax.sharding import ( _get_mesh, @@ -85,11 +85,13 @@ def _ep_domain_for_rank(mesh, ep_resource, rank, device_to_rank=None): def device_to_rank(d): return d.process_index - ep_pos = mesh.axis_names.index(ep_resource) - ep_size = mesh.shape[ep_resource] + ep_axes = _normalize_ep_axes(ep_resource) + ep_positions = tuple(mesh.axis_names.index(axis) for axis in ep_axes) + non_ep_positions = tuple(i for i in range(len(mesh.axis_names)) if i not in ep_positions) + ep_size = int(np.prod([mesh.shape[axis] for axis in ep_axes])) ranks = np.vectorize(device_to_rank, otypes=[np.int64])(mesh.devices) - # Move ep last and flatten: each row is one domain (all non-ep coords fixed). - grid = np.moveaxis(ranks, ep_pos, -1).reshape(-1, ep_size) + # Move EP axes last in their declared order and flatten them into one domain. + grid = np.transpose(ranks, non_ep_positions + ep_positions).reshape(-1, ep_size) loc = np.argwhere(grid == rank) if loc.shape[0] != 1: raise ValueError( @@ -125,16 +127,15 @@ def ep_bootstrap( max_token_dtype=jnp.bfloat16, max_num_sms=0, drop_on_overflow=False, + ep_axes=None, ): """Initialize the EP communicator. Call once per process before any EP op. - Must run inside the active JAX Mesh and a global_shard_guard; ep_size and - num_ep_groups are read from the mesh axes named by MeshResource.ep_resource - and MeshResource.dp_resource/fsdp_resource. Axes orthogonal to EP (tp, pp, - cp, ...) are supported and replicated across EP tensors. Auto-selects - between self-hosted NCCL and XLA's borrowed comm (see - tex.ep.use_nccl_comm_from_xla). - Multiple local devices require the XLA borrowed communicator path. + Must run inside an active JAX Mesh. ``ep_axes`` overrides + ``MeshResource.ep_resource`` when provided; axes outside the ordered EP + tuple form separate EP domains. The communicator is auto-selected between + self-hosted NCCL and XLA's borrowed comm (see + ``tex.ep.use_nccl_comm_from_xla``). Args: world_size: Total number of devices (product of all mesh axes). @@ -152,6 +153,8 @@ def ep_bootstrap( drop_on_overflow: Drop tokens exceeding recv_capacity_per_rank instead of trapping on overflow. Dropped tokens are still counted in total_recv_tokens, so callers can detect overflow from it. + ep_axes: Physical mesh axis name or ordered tuple of names. Defaults to + ``MeshResource.ep_resource``. """ if rank is None: rank = jax.process_index() @@ -167,13 +170,12 @@ def ep_bootstrap( " XLA collectives FFI support, or launch one process per GPU instead." ) - gsr = global_mesh_resource() - ep_resource = gsr.ep_resource + explicit_ep_axes = ep_axes + ep_resource = ( + global_mesh_resource().ep_resource if explicit_ep_axes is None else explicit_ep_axes + ) if ep_resource is None: - raise ValueError( - "ep_bootstrap requires MeshResource.ep_resource to be set; enter a" - " global_shard_guard(MeshResource(..., ep_resource=)) before bootstrap." - ) + raise ValueError("ep_bootstrap requires ep_axes or MeshResource.ep_resource to be set.") mesh = _get_mesh() if mesh.empty: raise ValueError( @@ -192,11 +194,19 @@ def ep_bootstrap( f"ep_bootstrap: mesh device count ({get_num_devices_in_mesh(mesh)}) must equal" f" world_size ({world_size})." ) - ep_size = get_mesh_axis_size(ep_resource) + ep_axes = _normalize_ep_axes(ep_resource) + ep_size = int(np.prod([mesh.shape[axis] for axis in ep_axes])) # num_ep_groups counts only the distinct-token (dp/fsdp) axes; replicated # axes (tp, pp, ...) do not create distinct EP-output slabs. - outer_axis = _ep_outer_axis() - num_ep_groups = 1 if outer_axis is None else get_mesh_axis_size(outer_axis) + resource = global_mesh_resource() + outer_axes = tuple( + dict.fromkeys( + a + for a in (resource.dp_resource, resource.fsdp_resource) + if a is not None and a not in ep_axes + ) + ) + num_ep_groups = int(np.prod([get_mesh_axis_size(a) for a in outer_axes])) if num_experts % ep_size != 0: raise ValueError(f"num_experts ({num_experts}) must be divisible by ep_size ({ep_size}).") @@ -298,24 +308,20 @@ def ep_finalize(): tex.ep.reset_ep_config() -def _default_out_partition_spec(): +def _default_out_partition_spec(ep_axes=None): """Leading-axis default: ``(("dp","ep"),)`` if DP/FSDP is set, else ``("ep",)``.""" - gsr = global_mesh_resource() - if gsr.ep_resource is None: - raise ValueError( - "ep_resource is not set on the active MeshResource; pass out_sharding=... explicitly." - ) - outer = _ep_outer_axis() + outer = _ep_outer_axis(ep_axes) outer_axes = (outer,) if isinstance(outer, str) else outer or () - leading = (*outer_axes, gsr.ep_resource) if outer_axes else gsr.ep_resource + leading_axes = (*outer_axes, *tex.ep._resolve_ep_axes(ep_axes)) + leading = leading_axes[0] if len(leading_axes) == 1 else leading_axes return (leading,) # ── ep_dispatch (custom_vjp) ───────────────────────────────────────────────── -@partial(jax.custom_vjp, nondiff_argnums=(0, 4)) -def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank): +@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5)) +def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=None): """Scatter tokens and weights to expert ranks. ``cfg`` is a per-layer ``EpLayerConfig``; distinct layers may share a @@ -330,28 +336,33 @@ def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank): ``drop_on_overflow`` is set. When ``recv_capacity_per_rank`` is not sized for the worst case, detect overflow by ``process_allgather``-ing it, then ``max(...) > recv_capacity_per_rank`` flags an overflowing step. + + ``ep_axes`` may be a mesh axis name or an ordered tuple of names. When it + is ``None``, the active ``MeshResource.ep_resource`` is used. """ - return _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank)[0] + return _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes)[0] -def _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank): +def _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=None): if not jnp.issubdtype(topk_weights.dtype, jnp.floating): raise TypeError( f"ep_dispatch: topk_weights must be a floating dtype; got {topk_weights.dtype}." ) recv_tokens, recv_topk_weights, token_counts, total_recv_tokens, handle_mem = ( - tex.ep_prepare_and_dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank) + tex.ep_prepare_and_dispatch_fwd( + cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=ep_axes + ) ) out_leading = tuple(tokens.shape[:-1]) primal = (recv_tokens, recv_topk_weights, handle_mem, token_counts, total_recv_tokens) return primal, (handle_mem, out_leading) -def _dispatch_bwd(cfg, recv_capacity_per_rank, res, g_outputs): +def _dispatch_bwd(cfg, recv_capacity_per_rank, ep_axes, res, g_outputs): del recv_capacity_per_rank handle_mem, out_leading = res # Re-pin cotangent: XLA transpose can drop the EP axis and feed the FFI a global tensor. - out_spec = _default_out_partition_spec() + out_spec = _default_out_partition_spec(ep_axes) spec = jax.sharding.PartitionSpec(*out_spec) g_recv_tokens = with_sharding_constraint(g_outputs[0], spec) g_recv_topk_weights = with_sharding_constraint(g_outputs[1], spec) @@ -362,6 +373,7 @@ def _dispatch_bwd(cfg, recv_capacity_per_rank, res, g_outputs): g_recv_topk_weights, out_leading, out_partition_spec=out_spec, + ep_axes=ep_axes, ) return (None, grad_tokens, grad_topk_weights) @@ -372,7 +384,7 @@ def _dispatch_bwd(cfg, recv_capacity_per_rank, res, g_outputs): # ── ep_combine (custom_vjp) ────────────────────────────────────────────────── -@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5)) +@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5, 6)) def ep_combine( cfg, handle_mem, @@ -380,6 +392,7 @@ def ep_combine( expert_out, num_local_tokens, out_sharding=None, + ep_axes=None, ): """Scatter-sum expert outputs back to source ranks. **Unweighted.** @@ -388,6 +401,8 @@ def ep_combine( not through this op. ``num_local_tokens`` is STATIC: int -> ``[T, H]``, tuple -> ``[*tuple, H]``. ``out_sharding`` defaults via ``_default_out_partition_spec``; only the leading dim may be sharded. + ``ep_axes`` accepts an explicit mesh axis name or tuple and otherwise falls + back to ``MeshResource.ep_resource``. """ return _combine_fwd( cfg, @@ -396,6 +411,7 @@ def ep_combine( expert_out, num_local_tokens, out_sharding, + ep_axes, )[0] @@ -406,24 +422,32 @@ def _combine_fwd( expert_out, num_local_tokens, out_sharding, + ep_axes=None, ): del token_counts if out_sharding is None: - out_sharding = _default_out_partition_spec() + out_sharding = _default_out_partition_spec(ep_axes) result = tex.ep_combine_fwd( - cfg, handle_mem, expert_out, num_local_tokens, out_partition_spec=out_sharding + cfg, + handle_mem, + expert_out, + num_local_tokens, + out_partition_spec=out_sharding, + ep_axes=ep_axes, ) return result, (handle_mem, expert_out.shape[-2]) -def _combine_bwd(cfg, _num_local_tokens, _out_sharding, res, g_result): +def _combine_bwd(cfg, _num_local_tokens, _out_sharding, ep_axes, res, g_result): handle_mem, recv_capacity_per_rank = res # Re-pin cotangent (same XLA-transpose workaround as _dispatch_bwd). if _out_sharding is None: - _out_sharding = _default_out_partition_spec() + _out_sharding = _default_out_partition_spec(ep_axes) spec = jax.sharding.PartitionSpec(*_out_sharding) g_result = with_sharding_constraint(g_result, spec) - grad_expert_out = tex.ep_combine_bwd(cfg, handle_mem, g_result, recv_capacity_per_rank) + grad_expert_out = tex.ep_combine_bwd( + cfg, handle_mem, g_result, recv_capacity_per_rank, ep_axes=ep_axes + ) return (None, None, grad_expert_out) diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index add9769700..89e2f501ba 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -92,6 +92,8 @@ class _MoEBlock(TransformerEngineBase): Logical sharding axis tuples (consumed by Flax's :func:`with_logical_partitioning` and our internal :func:`with_sharding_constraint_by_logical_axes`). + ep_axis : Optional[str | tuple[str, ...]] + Physical EP axis or ordered tuple of axes, overriding the EP resource. mesh_resource : Optional[MeshResource] Physical DP, FSDP and EP mesh axes. ``None`` resolves the active global MeshResource context. An explicit resource takes precedence; one of @@ -163,6 +165,7 @@ class _MoEBlock(TransformerEngineBase): input_axes: Tuple[Optional[str], ...] = () # Parallelism + ep_axis: Optional[Union[str, Tuple[str, ...]]] = None mesh_resource: Optional[MeshResource] = None quant_before_fsdp_ag: bool = False @@ -218,6 +221,7 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( self.mesh_resource, self.quant_before_fsdp_ag, + ep_axis=self.ep_axis, ) _, data_parallelism_axes = _moe_mesh_axes(mesh_resource) assert ( diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index a9c5366e0b..1a6abd434a 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -26,7 +26,7 @@ Sharding model -------------- * Inbound activations are 3D ``[B, S, H]`` sharded - ``((*data_parallelism_axes, ep_axis), None, None)``. The public + ``((outer_dp_axes..., ep_axes...), None, None)``. The public :func:`moe` soft-repins this on entry and warns when a reshard is inserted. * The EP, grouped-quantize, and grouped-GEMM primitives operate at global @@ -162,7 +162,10 @@ def _num_experts_per_tok(router_info: Union[RouterComputationInfo, RoutingMapInf def _moe_mesh_axes(resource: MeshResource): """Resolve physical MoE axes, keeping EP innermost in the batch shard.""" - if not isinstance(resource.ep_resource, str) or not resource.ep_resource: + if resource.ep_resource is None: + raise ValueError("TE MoE requires MeshResource.ep_resource to name physical mesh axes.") + ep_axes = tex.ep._normalize_ep_axes(resource.ep_resource) + if not ep_axes: raise ValueError("TE MoE requires MeshResource.ep_resource to name a physical mesh axis.") outer_axes = tuple( dict.fromkeys( @@ -171,7 +174,7 @@ def _moe_mesh_axes(resource: MeshResource): ) if any(not isinstance(axis, str) or not axis for axis in outer_axes): raise ValueError("TE MoE DP and FSDP resources must name physical mesh axes.") - if resource.ep_resource in outer_axes: + if any(axis in outer_axes for axis in ep_axes): raise ValueError("TE MoE EP and outer data-parallel axes must be distinct.") return resource.ep_resource, outer_axes @@ -179,6 +182,7 @@ def _moe_mesh_axes(resource: MeshResource): def _resolve_moe_mesh_resource( mesh_resource=None, quant_before_fsdp_ag=False, + ep_axis=None, ): """Resolve and snapshot the explicit or active mesh resource.""" if not isinstance(quant_before_fsdp_ag, bool): @@ -197,7 +201,11 @@ def _resolve_moe_mesh_resource( " context." ) # Snapshot the mutable resource so the custom VJP retains its forward-time axes. - mesh_resource = replace(mesh_resource) + mesh_resource = ( + replace(mesh_resource, ep_resource=ep_axis) + if ep_axis is not None + else replace(mesh_resource) + ) _moe_mesh_axes(mesh_resource) if quant_before_fsdp_ag and not mesh_resource.fsdp_resource: raise ValueError("quant_before_fsdp_ag=True requires MeshResource.fsdp_resource.") @@ -299,8 +307,14 @@ def _cudnn_jax_fusion_rejection_reasons( errors.append(f"unsupported MXFP8 payload dtype {quantizer.q_dtype} for {name}") mesh = _get_mesh() - if mesh is not None and not mesh.empty and ep_axis in mesh.shape: - num_local_experts = num_experts // mesh.shape[ep_axis] + if ( + mesh is not None + and not mesh.empty + and all(a in mesh.shape for a in tex.ep._normalize_ep_axes(ep_axis)) + ): + num_local_experts = num_experts // math.prod( + mesh.shape[a] for a in tex.ep._normalize_ep_axes(ep_axis) + ) if num_local_experts > 1024: errors.append(f"requires at most 1024 local experts, got {num_local_experts}") @@ -411,7 +425,7 @@ def _constraint_bwd(dtype_ref, grad): def _prepare_moe_input( x: jnp.ndarray, - ep_axis: str, + ep_axis: Union[str, Tuple[str, ...]], data_parallelism_axes: Tuple[str, ...], *, caller: str, @@ -419,9 +433,9 @@ def _prepare_moe_input( """Validate and constrain an MoE input to the EP communication layout. NCCL EP forms communication groups from consecutive global ranks, with - ``ep_axis`` as the innermost component of the compound leading mesh axis. + the EP axes as the innermost components of the compound leading mesh axis. The expected activation layout is therefore - ``P((*data_parallelism_axes, ep_axis), None, None)``. If ``x`` has an + ``P((*data_parallelism_axes, *ep_axes), None, None)``. If ``x`` has an incompatible known sharding, this function warns before inserting the reshard. The custom constraint also keeps the backward cotangent in the input activation dtype. @@ -430,8 +444,8 @@ def _prepare_moe_input( ---------- x : jnp.ndarray Rank-3 MoE input activation. - ep_axis : str - Physical mesh axis used for expert parallelism. + ep_axis : Union[str, Tuple[str, ...]] + Physical mesh axis, or ordered tuple of axes, used for expert parallelism. data_parallelism_axes : Tuple[str, ...] Physical outer data-parallel mesh axes. caller : str @@ -445,11 +459,12 @@ def _prepare_moe_input( mesh = _get_mesh() if mesh is None or mesh.empty: raise ValueError(f"{caller}(...) requires an active jax.sharding.Mesh.") - for axis in (ep_axis, *data_parallelism_axes): + ep_axes = tex.ep._normalize_ep_axes(ep_axis) + for axis in (*ep_axes, *data_parallelism_axes): if axis not in mesh.shape: raise ValueError(f"TE MoE resource axis {axis!r} is not in the active mesh.") - expected_leading: Any = (*data_parallelism_axes, ep_axis) if data_parallelism_axes else ep_axis + expected_leading: Any = (*data_parallelism_axes, *ep_axes) if data_parallelism_axes else ep_axis expected_spec = P(expected_leading, None, None) actual_spec = getattr(getattr(x, "sharding", None), "spec", None) if actual_spec is not None and tuple(actual_spec) != tuple(expected_spec): @@ -1376,7 +1391,7 @@ def _moe_fwd_rule( raise ValueError("moe(...) requires an active jax.sharding.Mesh.") if ep_axis is None: raise ValueError("moe(...) requires ep_axis to be set (TE EP backend).") - num_ep = mesh.shape[ep_axis] + num_ep = math.prod(mesh.shape[a] for a in tex.ep._normalize_ep_axes(ep_axis)) if num_experts % num_ep != 0: raise ValueError(f"num_experts={num_experts} must be divisible by EP size={num_ep}") num_local_experts = num_experts // num_ep @@ -1470,7 +1485,7 @@ def _moe_fwd_rule( # ep must be innermost: ep_bootstrap forms NCCL EP comms from # consecutive global ranks (dp_color = rank // ep_size), so the # comm only stays within one model replica under (outer_dp, ep). - batch_pspec_axis = (*data_parallelism_axes, ep_axis) + batch_pspec_axis = (*data_parallelism_axes, *tex.ep._normalize_ep_axes(ep_axis)) ep3_spec = P(batch_pspec_axis, None, None) ep2_spec = P(batch_pspec_axis, None) x = jax.lax.with_sharding_constraint(x, NamedSharding(mesh, ep3_spec)) @@ -1835,7 +1850,7 @@ def _moe_bwd_rule( if not data_parallelism_axes: batch_pspec_axis: Any = ep_axis else: - batch_pspec_axis = (*data_parallelism_axes, ep_axis) + batch_pspec_axis = (*data_parallelism_axes, *tex.ep._normalize_ep_axes(ep_axis)) ep3_spec = P(batch_pspec_axis, None, None) ep2_spec = P(batch_pspec_axis, None) out_partition_spec = (batch_pspec_axis, None, None) @@ -2169,6 +2184,7 @@ def moe( noop_quantizer_set, noop_quantizer_set, ), + ep_axis: Optional[Union[str, Tuple[str, ...]]] = None, mesh_resource: Optional[MeshResource] = None, quant_before_fsdp_ag: bool = False, input_axes: Tuple[Optional[str], ...] = (), @@ -2231,6 +2247,9 @@ def moe( dispatch_checkpoint_name : Optional[str] JAX rematerialization checkpoint name for the EP dispatch outputs. ``None`` leaves these values unnamed; the opaque prepare handle is never named. + ep_axis : Optional[str | tuple[str, ...]] + Override the expert-parallel resource with a physical axis or ordered + tuple of axes. All selected axes participate in one EP domain. mesh_resource : Optional[MeshResource] Physical parallelism resources. ``None`` uses the active ``global_shard_guard`` context; an explicit resource takes precedence. @@ -2289,7 +2308,7 @@ def moe( if dispatch_quantizer is not None and not isinstance(dispatch_quantizer, Quantizer): raise TypeError("dispatch_quantizer must be a Quantizer or None") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( - mesh_resource, quant_before_fsdp_ag + mesh_resource, quant_before_fsdp_ag, ep_axis=ep_axis ) ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource) From 405b3d9041d91ba5fdeb22f975b474c2c82133b5 Mon Sep 17 00:00:00 2001 From: Jeremy Berchtold Date: Thu, 17 Sep 2026 15:21:26 -0700 Subject: [PATCH 02/15] [JAX] Preserve FSDP with explicit compound EP axes --- transformer_engine/jax/cpp_extensions/ep.py | 2 -- transformer_engine/jax/ep.py | 12 +++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/transformer_engine/jax/cpp_extensions/ep.py b/transformer_engine/jax/cpp_extensions/ep.py index 9c7a036473..2d0acc440d 100644 --- a/transformer_engine/jax/cpp_extensions/ep.py +++ b/transformer_engine/jax/cpp_extensions/ep.py @@ -315,8 +315,6 @@ def _leading_axis_ok(spec, resource_axes=None, mesh=None, *, ep_axes=None): def _ep_outer_axis(ep_axes=None): """Legacy single outer axis used by the standalone EP custom VJP wrapper.""" - if ep_axes is not None: - return None resource = global_mesh_resource() ep_axes = _resolve_ep_axes(ep_axes) dp_resource = None if resource.dp_resource in ep_axes else resource.dp_resource diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 295aee4372..d5cf924633 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -105,15 +105,17 @@ def device_to_rank(d): def _ep_flattened_replica_groups(mesh, ep_resource): """FLATTENED_ID replica groups for the EP axis, as a flat int64 array. - Each group fixes all non-ep mesh coordinates and varies ep. Returns - ``(flat_groups, ep_size)``. + Each group fixes all non-EP mesh coordinates and varies the ordered EP + axes. Returns ``(flat_groups, ep_size)``. """ shape = tuple(mesh.shape[a] for a in mesh.axis_names) - ep_pos = mesh.axis_names.index(ep_resource) - ep_size = shape[ep_pos] + ep_axes = _normalize_ep_axes(ep_resource) + ep_pos = tuple(mesh.axis_names.index(axis) for axis in ep_axes) + non_ep_pos = tuple(i for i in range(len(shape)) if i not in ep_pos) + ep_size = int(np.prod([shape[i] for i in ep_pos])) world = int(np.prod(shape)) grid = np.arange(world, dtype=np.int64).reshape(shape) - groups = np.moveaxis(grid, ep_pos, -1).reshape(-1, ep_size) + groups = np.transpose(grid, non_ep_pos + ep_pos).reshape(-1, ep_size) return groups.reshape(-1), ep_size From 0df8099eb8c56f173d4bf01eb78f0bedc6c5a899 Mon Sep 17 00:00:00 2001 From: Jeremy Berchtold Date: Tue, 22 Sep 2026 11:03:31 -0700 Subject: [PATCH 03/15] Support 2D inputs with compound EP in JAX MoE --- tests/jax/run_te_ep_moe.sh | 3 +- tests/jax/test_te_ep_moe.py | 113 +++++++++++++++++------------ transformer_engine/jax/flax/moe.py | 10 ++- transformer_engine/jax/moe.py | 19 ++++- 4 files changed, 91 insertions(+), 54 deletions(-) diff --git a/tests/jax/run_te_ep_moe.sh b/tests/jax/run_te_ep_moe.sh index ee1c55fc33..e3c081831c 100755 --- a/tests/jax/run_te_ep_moe.sh +++ b/tests/jax/run_te_ep_moe.sh @@ -8,7 +8,7 @@ # its own --num-process=N --process-id=i, and waits for all of them. Each # child calls jax.distributed.initialize(..., local_device_ids=process_id) # so each Python process only sees its one GPU as a local device and the -# participating processes form a global (ep, fsdp) mesh. +# participating processes form a global (fsdp, expert[, tensor]) mesh. set -euo pipefail @@ -38,6 +38,7 @@ echo "============================================================" echo "TE-EP MoE MULTIPROCESS test (one process per GPU, ${NUM_GPUS} GPUs)" echo " test file : $TEST_FILE" echo " coordinator : $TE_EP_MOE_COORDINATOR_ADDRESS" +echo " TP size / input rank: ${TE_EP_MOE_TP_SIZE:-2} / ${TE_EP_MOE_INPUT_RANK:-auto}" echo " XLA_PYTHON_CLIENT_PREALLOCATE: $XLA_PYTHON_CLIENT_PREALLOCATE" echo " XLA_PYTHON_CLIENT_MEM_FRACTION: $XLA_PYTHON_CLIENT_MEM_FRACTION" if [ "$#" -gt 0 ]; then diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 71535d2f93..f0416170cf 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -7,7 +7,7 @@ The launcher ``tests/jax/run_te_ep_moe.sh`` forks one pytest process per visible GPU. Each process binds to exactly one device via ``jax.distributed.initialize(..., local_device_ids=process_id)``; the -participating processes form a global ``(ep, fsdp)`` mesh through JAX's +participating processes form a global ``(fsdp, expert, tensor)`` mesh through JAX's distributed runtime. How to run @@ -43,6 +43,7 @@ main+aux grads stay finite) in two consolidated tests. """ +import math import importlib import os @@ -111,12 +112,11 @@ def _read_mp_options(): from transformer_engine_jax import get_device_compute_capability -# Grouped GEMM in the MoE custom_vjp requires Blackwell (sm_100+). The -# TE EP NCCL primitives themselves need SM>=90, but the FFN body uses -# grouped_gemm, so the file as a whole gates on sm_100+. -if get_device_compute_capability(0) < 100: +# The TE EP NCCL primitives and the grouped-GEMM fallback used by the MoE +# custom_vjp are supported on Hopper (sm_90+) and newer GPUs. +if get_device_compute_capability(0) < 90: pytest.skip( - "MoE TE EP tests require Blackwell (sm_100+) for grouped GEMM", + "MoE TE EP tests require Hopper (sm_90+) or newer", allow_module_level=True, ) @@ -136,28 +136,40 @@ def _read_mp_options(): # Mesh / shape config # ----------------------------------------------------------------------------- -EP_AXIS = "ep" +EXPERT_AXIS = "expert" +TENSOR_AXIS = "tensor" FSDP_AXIS = "fsdp" EP_SIZE = 2 -assert ( - jax.device_count() % EP_SIZE == 0 -), f"device_count {jax.device_count()} must be divisible by EP_SIZE={EP_SIZE}" -FSDP_SIZE = jax.device_count() // EP_SIZE -NUM_DEVICES_REQUIRED = EP_SIZE * FSDP_SIZE +TP_SIZE = int(os.environ.get("TE_EP_MOE_TP_SIZE", "2")) +INPUT_RANK = int(os.environ.get("TE_EP_MOE_INPUT_RANK", "2" if TP_SIZE > 1 else "3")) +FSDP_SIZE = jax.device_count() // (EP_SIZE * TP_SIZE) +assert TP_SIZE in (1, 2), f"TE_EP_MOE_TP_SIZE must be 1 or 2, got {TP_SIZE}" +assert INPUT_RANK in (2, 3), f"TE_EP_MOE_INPUT_RANK must be 2 or 3, got {INPUT_RANK}" +EP_AXIS = EXPERT_AXIS if TP_SIZE == 1 else (EXPERT_AXIS, TENSOR_AXIS) +EP_AXES = (EP_AXIS,) if isinstance(EP_AXIS, str) else EP_AXIS +EXPLICIT_EP_AXIS = None if TP_SIZE == 1 else EP_AXIS +MESH_SHAPE = (FSDP_SIZE, EP_SIZE) if TP_SIZE == 1 else (FSDP_SIZE, EP_SIZE, TP_SIZE) +MESH_AXES = (FSDP_AXIS, *EP_AXES) +COMBINED_EP_SIZE = EP_SIZE * TP_SIZE +NUM_DEVICES_REQUIRED = FSDP_SIZE * EP_SIZE * TP_SIZE +assert jax.device_count() == NUM_DEVICES_REQUIRED, ( + f"device_count {jax.device_count()} must equal {NUM_DEVICES_REQUIRED} for " + f"expert={EP_SIZE} x fsdp={FSDP_SIZE} x tensor={TP_SIZE}" +) LOGICAL_AXIS_RULES = ( - ("exp", EP_AXIS), - ("expert_weight_fsdp", (EP_AXIS, FSDP_AXIS)), + ("exp", EP_AXES), + ("expert_weight_fsdp", (*EP_AXES, FSDP_AXIS)), ("gated_weight_fsdp", FSDP_AXIS), ("embed", FSDP_AXIS), ("mlp", None), - ("batch", (FSDP_AXIS, EP_AXIS)), + ("batch", (FSDP_AXIS, *EP_AXES)), ) # Small shapes so the parity tests stay tight on bf16. The block still -# has all four ranks participating in dispatch/combine. +# has all eight ranks participating in dispatch/combine. DTYPE = jnp.bfloat16 -BATCH = EP_SIZE * FSDP_SIZE * 2 # 8 on 4-GPU, 16 on 8-GPU +BATCH = NUM_DEVICES_REQUIRED * 2 SEQ = 32 HIDDEN = 512 # multiple of 512 so MXFP8 dispatch scale rows are 16-byte aligned INTER = 128 @@ -201,14 +213,14 @@ def use_cudnn_fusion(request): def mesh(use_cudnn_fusion): if jax.device_count() < NUM_DEVICES_REQUIRED: pytest.skip( - f"Need >={NUM_DEVICES_REQUIRED} devices for ep={EP_SIZE} x fsdp={FSDP_SIZE};" + f"Need {NUM_DEVICES_REQUIRED} devices for expert={EP_SIZE} x " + f"fsdp={FSDP_SIZE} x tensor={TP_SIZE};" f" have {jax.device_count()}" ) - # ``ep`` must be the inner axis: ``ep_bootstrap`` forms NCCL EP groups - # from consecutive global ranks via ``dp_color = rank // ep_size``, so - # only an (outer_fsdp, inner_ep) device layout groups ranks correctly. - devices = mesh_utils.create_device_mesh((FSDP_SIZE, EP_SIZE)) - mesh_obj = Mesh(devices, axis_names=(FSDP_AXIS, EP_AXIS)) + # Both ``expert`` and ``tensor`` act as EP axes. Keep the compound EP + # axes innermost so each FSDP replica owns one consecutive NCCL EP group. + devices = mesh_utils.create_device_mesh(MESH_SHAPE) + mesh_obj = Mesh(devices, axis_names=MESH_AXES) num_procs = jax.process_count() max_tokens_per_rank = (BATCH // num_procs) * SEQ @@ -219,14 +231,16 @@ def mesh(use_cudnn_fusion): num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, max_tokens_per_rank=max_tokens_per_rank, - ep_size=EP_SIZE, + ep_size=COMBINED_EP_SIZE, use_cudnn_fusion=use_cudnn_fusion, ) # Eager bootstrap: ep_bootstrap does a host-side NCCL UID allgather # and cannot run from inside jax.jit. Sized to the worst-case recv_pr # across _CONFIGS so every parametrized config is bootstrap-compatible. - with mesh_obj, global_shard_guard(MeshResource(ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS)): + with mesh_obj, global_shard_guard( + MeshResource(ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS) + ): ep_bootstrap( world_size=num_procs, rank=jax.process_index(), @@ -235,13 +249,14 @@ def mesh(use_cudnn_fusion): recv_capacity_per_rank=recv_capacity_per_rank, hidden_dim=HIDDEN, max_token_dtype=DTYPE, + ep_axes=EXPLICIT_EP_AXIS, ) record_ep_bootstrap_signature_for_moe( num_experts=NUM_EXPERTS, max_tokens_per_rank=max_tokens_per_rank, recv_capacity_per_rank=recv_capacity_per_rank, hidden_dim=HIDDEN, - ep_size=EP_SIZE, + ep_size=COMBINED_EP_SIZE, ) mode = "cudnn" if use_cudnn_fusion else "unfused" try: @@ -299,8 +314,9 @@ def _pure_jax_moe_reference( score_function: str = "softmax", scaling_factor: float = 1.0, ): - B, S, H = x.shape - T = B * S + original_shape = x.shape + H = original_shape[-1] + T = math.prod(original_shape[:-1]) K = num_experts_per_tok x_2d = x.reshape(T, H) @@ -351,7 +367,7 @@ def _pure_jax_moe_reference( intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) - output = output_2d.reshape(B, S, H).astype(x.dtype) + output = output_2d.reshape(original_shape).astype(x.dtype) if aux_loss_coeff > 0.0: if hash_input_ids is not None: @@ -396,7 +412,7 @@ def _make_block( score_function="softmax", scaling_factor=1.0, expert_bias_init=None, - input_axes=("batch", None, None), + input_axes=None, quantization_recipe=None, dispatch_checkpoint_name=None, quant_before_fsdp_ag=False, @@ -404,10 +420,13 @@ def _make_block( dispatch_quantization_recipe=None, use_cudnn_fusion, ): + if input_axes is None: + input_axes = ("batch", *([None] * (INPUT_RANK - 1))) kwargs = dict( num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, + ep_axis=EXPLICIT_EP_AXIS, mesh_resource=mesh_resource, apply_topk_weights_early=apply_topk_weights_early, aux_loss_coeff=aux_loss_coeff, @@ -448,9 +467,8 @@ def _strong_expert_bias_init(key, shape, dtype): def _shard_inputs(x, mesh): # Match the layout moe.py re-pins to: outer dp axes, then ep innermost. - return jax.lax.with_sharding_constraint( - x, NamedSharding(mesh, P((FSDP_AXIS, EP_AXIS), None, None)) - ) + spec = P((FSDP_AXIS, *EP_AXES), *([None] * (x.ndim - 1))) + return jax.lax.with_sharding_constraint(x, NamedSharding(mesh, spec)) def _ctx(mesh): @@ -460,7 +478,7 @@ class _Combo: def __enter__(self_inner): self_inner._m = mesh.__enter__() self_inner._gs = global_shard_guard( - MeshResource(ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) + MeshResource(ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS) ) self_inner._gs.__enter__() self_inner._ar = nn_partitioning.axis_rules(LOGICAL_AXIS_RULES) @@ -556,7 +574,8 @@ def _params_global_numpy(variables, mesh): def _make_inputs(key): """Generate a globally-identical input tensor on every process.""" - return jax.random.normal(key, (BATCH, SEQ, HIDDEN), dtype=DTYPE) + shape = (BATCH * SEQ, HIDDEN) if INPUT_RANK == 2 else (BATCH, SEQ, HIDDEN) + return jax.random.normal(key, shape, dtype=DTYPE) def _quantization_recipe(quantization): @@ -613,7 +632,10 @@ def _quantization_recipe(quantization): pytest.param("bf16", id="bf16"), ] -if get_device_compute_capability(0) >= 100: +if ( + get_device_compute_capability(0) >= 90 + and os.environ.get("NVTE_JAX_ENFORCE_V2_GROUPED_GEMM", "0") != "1" +): _QUANTIZATION_CASES.append(pytest.param("mxfp8", id="mxfp8")) @@ -651,7 +673,7 @@ def layout_moe(*args, **kwargs): if fsdp_dimension == "expert": kwargs["wi_kernel_axes"] = ("expert_weight_fsdp", None, None) kwargs["wo_kernel_axes"] = ("expert_weight_fsdp", None, None) - spec = P((EP_AXIS, FSDP_AXIS), None, None) + spec = P((*EP_AXES, FSDP_AXIS), None, None) args[2] = jax.lax.with_sharding_constraint(args[2], NamedSharding(mesh, spec)) args[3] = jax.lax.with_sharding_constraint(args[3], NamedSharding(mesh, spec)) return original_moe(*args, **kwargs) @@ -794,7 +816,8 @@ class TestTeEpMoeHashRouting: def test_hash_forward_and_backward(self, mesh, make_block, use_cudnn_fusion): vocab_size = 64 scaling_factor = 1.5 - token_ids = jnp.arange(BATCH * SEQ, dtype=jnp.int32).reshape(BATCH, SEQ) % vocab_size + x = _make_inputs(jax.random.PRNGKey(30)) + token_ids = jnp.arange(BATCH * SEQ, dtype=jnp.int32).reshape(x.shape[:-1]) % vocab_size vocab_ids = jnp.arange(vocab_size, dtype=jnp.int32) tid2eid = jnp.stack( [vocab_ids % NUM_EXPERTS, (vocab_ids + 3) % NUM_EXPERTS], @@ -804,14 +827,10 @@ def test_hash_forward_and_backward(self, mesh, make_block, use_cudnn_fusion): score_function="sqrtsoftplus", scaling_factor=scaling_factor, ) - x = _make_inputs(jax.random.PRNGKey(30)) with _ctx(mesh): x_sh = _shard_inputs(x, mesh) - ids_sh = jax.device_put( - token_ids, - NamedSharding(mesh, P((FSDP_AXIS, EP_AXIS), None)), - ) + ids_sh = _shard_inputs(token_ids, mesh) table_sh = jax.device_put(tid2eid, NamedSharding(mesh, P())) variables = jax.jit(block.init)(jax.random.PRNGKey(31), x_sh) params = { @@ -820,7 +839,7 @@ def test_hash_forward_and_backward(self, mesh, make_block, use_cudnn_fusion): def externally_routed_moe(params, activations, token_ids_arg, tid2eid_arg): logits = jnp.einsum( - "bsh,he->bse", + "...h,he->...e", activations, params["gate_kernel"].astype(activations.dtype), ).astype(jnp.float32) @@ -840,7 +859,8 @@ def externally_routed_moe(params, activations, token_ids_arg, tid2eid_arg): params["wi"], params["wo"], num_experts=NUM_EXPERTS, - input_axes=("batch", None, None), + ep_axis=EXPLICIT_EP_AXIS, + input_axes=("batch", *([None] * (INPUT_RANK - 1))), wi_kernel_axes=("exp", "embed", "mlp"), wo_kernel_axes=("exp", "mlp", "embed"), dtype=DTYPE, @@ -965,7 +985,8 @@ def functional_moe(params, activations): params["wi"], params["wo"], num_experts=NUM_EXPERTS, - input_axes=("batch", None, None), + ep_axis=EXPLICIT_EP_AXIS, + input_axes=("batch", *([None] * (INPUT_RANK - 1))), dtype=DTYPE, use_cudnn_fusion=use_cudnn_fusion, ) diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 89e2f501ba..573447ed88 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -94,6 +94,7 @@ class _MoEBlock(TransformerEngineBase): :func:`with_sharding_constraint_by_logical_axes`). ep_axis : Optional[str | tuple[str, ...]] Physical EP axis or ordered tuple of axes, overriding the EP resource. + Inputs may be [tokens, hidden] or [batch, sequence, hidden]. mesh_resource : Optional[MeshResource] Physical DP, FSDP and EP mesh axes. ``None`` resolves the active global MeshResource context. An explicit resource takes precedence; one of @@ -224,10 +225,11 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] ep_axis=self.ep_axis, ) _, data_parallelism_axes = _moe_mesh_axes(mesh_resource) - assert ( - inputs.ndim == 3 - ), f"_MoEBlock expects [batch, sequence, hidden] input, got shape {inputs.shape}" - _, _, hidden_size = inputs.shape + assert inputs.ndim in (2, 3), ( + "_MoEBlock expects [tokens, hidden] or [batch, sequence, hidden] input, got shape" + f" {inputs.shape}" + ) + hidden_size = inputs.shape[-1] # Param registrations -- must run OUTSIDE any JAX transform that # alters the variable scope (e.g. shard_map). The functional diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 1a6abd434a..a261477bc4 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -25,7 +25,7 @@ Sharding model -------------- -* Inbound activations are 3D ``[B, S, H]`` sharded +* Inbound activations are 2D ``[T, H]`` or 3D ``[B, S, H]`` sharded ``((outer_dp_axes..., ep_axes...), None, None)``. The public :func:`moe` soft-repins this on entry and warns when a reshard is inserted. @@ -141,7 +141,8 @@ class RoutingMapInfo: Parameters ---------- routing_indices : jnp.ndarray - Integer expert indices with shape ``[batch, sequence, top_k]``. + Integer expert indices with shape ``[batch, sequence, top_k]``, or + ``[tokens, top_k]`` for a ``[tokens, hidden]`` input. Non-differentiable. Each token's ``top_k`` indices must be distinct, otherwise the default receive-capacity bound is invalid. routing_weights : jnp.ndarray @@ -2310,6 +2311,18 @@ def moe( mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag, ep_axis=ep_axis ) + original_shape = x.shape + if x.ndim not in (2, 3): + raise ValueError(f"moe expects rank-2 or rank-3 input, got {x.shape}") + if x.ndim == 2: + x = x.reshape(x.shape[0], 1, x.shape[1]) + if input_axes: + input_axes = (input_axes[0], None, input_axes[1]) + if isinstance(router_info, RoutingMapInfo): + router_info = router_info.replace( + routing_indices=router_info.routing_indices[:, None], + routing_weights=router_info.routing_weights[:, None], + ) ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource) if isinstance(router_info, RouterComputationInfo): @@ -2400,4 +2413,4 @@ def moe( assert ( output.dtype == x.dtype ), f"moe() output dtype {output.dtype} != input dtype {x.dtype}" - return output, aux_loss, total_recv_tokens, expert_counts + return output.reshape(original_shape), aux_loss, total_recv_tokens, expert_counts From 0a8818151738fa8ba9b85de89e3c10652d30865f Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 16:37:53 -0800 Subject: [PATCH 04/15] Name compound EP axes only through MeshResource in MoE and ep_bootstrap Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 9 +++++++++ tests/jax/test_te_ep_moe.py | 23 +++++++++++++++++------ transformer_engine/jax/ep.py | 25 +++++++++++++------------ transformer_engine/jax/flax/moe.py | 11 +++-------- transformer_engine/jax/moe.py | 16 ++++------------ transformer_engine/jax/sharding.py | 5 +++-- 6 files changed, 49 insertions(+), 40 deletions(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 584dff60ad..62e8e8ece4 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -56,6 +56,15 @@ def test_global_default_and_duplicate_outer_axis(): assert quantize is False +def test_tuple_ep_resource(): + resource, _ = _resolve_moe_mesh_resource( + MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource=("ep", "tp")) + ) + assert _moe_mesh_axes(resource) == (("ep", "tp"), ("dp", "fsdp")) + with pytest.raises(ValueError, match="distinct"): + _resolve_moe_mesh_resource(MeshResource(fsdp_resource="tp", ep_resource=("ep", "tp"))) + + @pytest.mark.parametrize("fsdp_resource", ["fsdp", ("fsdp",), ("fsdp", "ep"), ("ep", "fsdp")]) def test_ep_partitioning_retains_bind_time_resources(fsdp_resource): from transformer_engine.jax.cpp_extensions import ep diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index f0416170cf..2e7db3f733 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -128,6 +128,7 @@ def _read_mp_options(): moe, record_ep_bootstrap_signature_for_moe, ) +from transformer_engine.jax.cpp_extensions.ep import get_ep_config from transformer_engine.jax.ep import ep_bootstrap, ep_finalize from transformer_engine.common.recipe import Format, MXFP8BlockScaling from transformer_engine.jax.sharding import MeshResource, global_shard_guard @@ -147,7 +148,10 @@ def _read_mp_options(): assert INPUT_RANK in (2, 3), f"TE_EP_MOE_INPUT_RANK must be 2 or 3, got {INPUT_RANK}" EP_AXIS = EXPERT_AXIS if TP_SIZE == 1 else (EXPERT_AXIS, TENSOR_AXIS) EP_AXES = (EP_AXIS,) if isinstance(EP_AXIS, str) else EP_AXIS -EXPLICIT_EP_AXIS = None if TP_SIZE == 1 else EP_AXIS +# With TP, the MoE block overrides the global resource so EP spans (expert, tensor). +EXPLICIT_MESH_RESOURCE = ( + None if TP_SIZE == 1 else MeshResource(ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) +) MESH_SHAPE = (FSDP_SIZE, EP_SIZE) if TP_SIZE == 1 else (FSDP_SIZE, EP_SIZE, TP_SIZE) MESH_AXES = (FSDP_AXIS, *EP_AXES) COMBINED_EP_SIZE = EP_SIZE * TP_SIZE @@ -249,7 +253,7 @@ def mesh(use_cudnn_fusion): recv_capacity_per_rank=recv_capacity_per_rank, hidden_dim=HIDDEN, max_token_dtype=DTYPE, - ep_axes=EXPLICIT_EP_AXIS, + mesh_resource=EXPLICIT_MESH_RESOURCE, ) record_ep_bootstrap_signature_for_moe( num_experts=NUM_EXPERTS, @@ -416,7 +420,7 @@ def _make_block( quantization_recipe=None, dispatch_checkpoint_name=None, quant_before_fsdp_ag=False, - mesh_resource=None, + mesh_resource=EXPLICIT_MESH_RESOURCE, dispatch_quantization_recipe=None, use_cudnn_fusion, ): @@ -426,7 +430,6 @@ def _make_block( num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, - ep_axis=EXPLICIT_EP_AXIS, mesh_resource=mesh_resource, apply_topk_weights_early=apply_topk_weights_early, aux_loss_coeff=aux_loss_coeff, @@ -707,6 +710,14 @@ def layout_moe(*args, **kwargs): ) +@pytest.mark.usefixtures("mesh") +def test_ep_bootstrap_matches_mesh_resource(): + """Bootstrap sizes EP from the MoE resource and counts every outer data-parallel axis.""" + cfg = get_ep_config() + assert cfg.ep_size == COMBINED_EP_SIZE + assert cfg.num_ep_groups == NUM_DEVICES_REQUIRED // COMBINED_EP_SIZE + + @pytest.mark.parametrize("quant_before_fsdp_ag", [False, True]) def test_mesh_resource_api_forward_and_backward(mesh, quant_before_fsdp_ag, make_block): """Explicit and global resources give identical forward and backward results.""" @@ -859,7 +870,7 @@ def externally_routed_moe(params, activations, token_ids_arg, tid2eid_arg): params["wi"], params["wo"], num_experts=NUM_EXPERTS, - ep_axis=EXPLICIT_EP_AXIS, + mesh_resource=EXPLICIT_MESH_RESOURCE, input_axes=("batch", *([None] * (INPUT_RANK - 1))), wi_kernel_axes=("exp", "embed", "mlp"), wo_kernel_axes=("exp", "mlp", "embed"), @@ -985,7 +996,7 @@ def functional_moe(params, activations): params["wi"], params["wo"], num_experts=NUM_EXPERTS, - ep_axis=EXPLICIT_EP_AXIS, + mesh_resource=EXPLICIT_MESH_RESOURCE, input_axes=("batch", *([None] * (INPUT_RANK - 1))), dtype=DTYPE, use_cudnn_fusion=use_cudnn_fusion, diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index d5cf924633..3d93ecb6a1 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -129,13 +129,13 @@ def ep_bootstrap( max_token_dtype=jnp.bfloat16, max_num_sms=0, drop_on_overflow=False, - ep_axes=None, + mesh_resource=None, ): """Initialize the EP communicator. Call once per process before any EP op. - Must run inside an active JAX Mesh. ``ep_axes`` overrides - ``MeshResource.ep_resource`` when provided; axes outside the ordered EP - tuple form separate EP domains. The communicator is auto-selected between + Must run inside an active JAX Mesh. ``mesh_resource`` names the EP and + outer DP/FSDP mesh axes, as for ``moe``; axes outside the ordered EP axes + form separate EP domains. The communicator is auto-selected between self-hosted NCCL and XLA's borrowed comm (see ``tex.ep.use_nccl_comm_from_xla``). @@ -155,8 +155,8 @@ def ep_bootstrap( drop_on_overflow: Drop tokens exceeding recv_capacity_per_rank instead of trapping on overflow. Dropped tokens are still counted in total_recv_tokens, so callers can detect overflow from it. - ep_axes: Physical mesh axis name or ordered tuple of names. Defaults to - ``MeshResource.ep_resource``. + mesh_resource: MeshResource naming the EP and outer DP/FSDP mesh axes. + ``None`` uses the active global_shard_guard. """ if rank is None: rank = jax.process_index() @@ -172,12 +172,14 @@ def ep_bootstrap( " XLA collectives FFI support, or launch one process per GPU instead." ) - explicit_ep_axes = ep_axes - ep_resource = ( - global_mesh_resource().ep_resource if explicit_ep_axes is None else explicit_ep_axes - ) + resource = global_mesh_resource() if mesh_resource is None else mesh_resource + ep_resource = resource.ep_resource if ep_resource is None: - raise ValueError("ep_bootstrap requires ep_axes or MeshResource.ep_resource to be set.") + raise ValueError( + "ep_bootstrap requires MeshResource.ep_resource to be set; pass" + " mesh_resource=MeshResource(..., ep_resource=) or enter a" + " global_shard_guard before bootstrap." + ) mesh = _get_mesh() if mesh.empty: raise ValueError( @@ -200,7 +202,6 @@ def ep_bootstrap( ep_size = int(np.prod([mesh.shape[axis] for axis in ep_axes])) # num_ep_groups counts only the distinct-token (dp/fsdp) axes; replicated # axes (tp, pp, ...) do not create distinct EP-output slabs. - resource = global_mesh_resource() outer_axes = tuple( dict.fromkeys( a diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 573447ed88..029a6d7a3e 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -92,14 +92,11 @@ class _MoEBlock(TransformerEngineBase): Logical sharding axis tuples (consumed by Flax's :func:`with_logical_partitioning` and our internal :func:`with_sharding_constraint_by_logical_axes`). - ep_axis : Optional[str | tuple[str, ...]] - Physical EP axis or ordered tuple of axes, overriding the EP resource. - Inputs may be [tokens, hidden] or [batch, sequence, hidden]. mesh_resource : Optional[MeshResource] Physical DP, FSDP and EP mesh axes. ``None`` resolves the active global MeshResource context. An explicit resource takes precedence; one of these is required. Batch sharding uses DP and FSDP as outer axes and - EP innermost. + EP innermost; EP may be an ordered tuple of axes. quant_before_fsdp_ag : bool Quantize MXFP8 weight shards before their FSDP all-gather. Defaults to ``False``; ``True`` requires ``mesh_resource.fsdp_resource``. @@ -166,7 +163,6 @@ class _MoEBlock(TransformerEngineBase): input_axes: Tuple[Optional[str], ...] = () # Parallelism - ep_axis: Optional[Union[str, Tuple[str, ...]]] = None mesh_resource: Optional[MeshResource] = None quant_before_fsdp_ag: bool = False @@ -204,12 +200,12 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] Parameters ---------- inputs : jnp.ndarray - ``[batch, sequence, hidden]``. + ``[batch, sequence, hidden]`` or ``[tokens, hidden]``. Returns ------- output : jnp.ndarray - ``[batch, sequence, hidden]``. + Same shape as ``inputs``. aux_loss : Optional[jnp.ndarray] Scalar load-balancing loss when ``aux_loss_coeff > 0``, else ``None``. @@ -222,7 +218,6 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( self.mesh_resource, self.quant_before_fsdp_ag, - ep_axis=self.ep_axis, ) _, data_parallelism_axes = _moe_mesh_axes(mesh_resource) assert inputs.ndim in (2, 3), ( diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index a261477bc4..a98e7f6260 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -183,7 +183,6 @@ def _moe_mesh_axes(resource: MeshResource): def _resolve_moe_mesh_resource( mesh_resource=None, quant_before_fsdp_ag=False, - ep_axis=None, ): """Resolve and snapshot the explicit or active mesh resource.""" if not isinstance(quant_before_fsdp_ag, bool): @@ -202,11 +201,7 @@ def _resolve_moe_mesh_resource( " context." ) # Snapshot the mutable resource so the custom VJP retains its forward-time axes. - mesh_resource = ( - replace(mesh_resource, ep_resource=ep_axis) - if ep_axis is not None - else replace(mesh_resource) - ) + mesh_resource = replace(mesh_resource) _moe_mesh_axes(mesh_resource) if quant_before_fsdp_ag and not mesh_resource.fsdp_resource: raise ValueError("quant_before_fsdp_ag=True requires MeshResource.fsdp_resource.") @@ -2185,7 +2180,6 @@ def moe( noop_quantizer_set, noop_quantizer_set, ), - ep_axis: Optional[Union[str, Tuple[str, ...]]] = None, mesh_resource: Optional[MeshResource] = None, quant_before_fsdp_ag: bool = False, input_axes: Tuple[Optional[str], ...] = (), @@ -2248,15 +2242,13 @@ def moe( dispatch_checkpoint_name : Optional[str] JAX rematerialization checkpoint name for the EP dispatch outputs. ``None`` leaves these values unnamed; the opaque prepare handle is never named. - ep_axis : Optional[str | tuple[str, ...]] - Override the expert-parallel resource with a physical axis or ordered - tuple of axes. All selected axes participate in one EP domain. mesh_resource : Optional[MeshResource] Physical parallelism resources. ``None`` uses the active ``global_shard_guard`` context; an explicit resource takes precedence. One of these is required. Batch sharding combines ``dp_resource`` then ``fsdp_resource`` (omitting unset/duplicate axes), with ``ep_resource`` - innermost. EP must be set and distinct from the outer axes. + innermost. EP must be set and distinct from the outer axes; it may be + an ordered tuple of axes that together form one EP domain. quant_before_fsdp_ag : bool Quantize expert-weight shards before gathering their MXFP8 data and scales on ``mesh_resource.fsdp_resource``. Default ``False`` gathers @@ -2309,7 +2301,7 @@ def moe( if dispatch_quantizer is not None and not isinstance(dispatch_quantizer, Quantizer): raise TypeError("dispatch_quantizer must be a Quantizer or None") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( - mesh_resource, quant_before_fsdp_ag, ep_axis=ep_axis + mesh_resource, quant_before_fsdp_ag ) original_shape = x.shape if x.ndim not in (2, 3): diff --git a/transformer_engine/jax/sharding.py b/transformer_engine/jax/sharding.py index 9fa1c77309..7ae94936ed 100644 --- a/transformer_engine/jax/sharding.py +++ b/transformer_engine/jax/sharding.py @@ -345,7 +345,8 @@ class MeshResource: compound with ``dp_resource`` / ``fsdp_resource`` as outer, e.g. ``PartitionSpec(("dp", "ep"), None, None)``). Dispatch output ``[ep_size, recv_capacity, H]`` is always sharded by ``ep_resource`` - on the leading ``ep_size`` dim. + on the leading ``ep_size`` dim. An ordered tuple of axis names, e.g. + ``("ep", "tp")``, makes those axes one EP domain. """ dp_resource: str = None @@ -354,7 +355,7 @@ class MeshResource: fsdp_resource: str | tuple[str, ...] = None pp_resource: str = None cp_resource: str = None - ep_resource: str = None + ep_resource: str | tuple[str, ...] = None _MeshResourceSnapshot = namedtuple("_MeshResourceSnapshot", MeshResource.__dataclass_fields__) From c2f1b6e238f0d20dee2db0957488521e3edc6e49 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 16:39:07 -0800 Subject: [PATCH 05/15] Flatten tuple fsdp resources and drop EP axes when counting EP groups Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 19 +++++++++++++++++++ transformer_engine/jax/ep.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 62e8e8ece4..a8547e684b 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -65,6 +65,25 @@ def test_tuple_ep_resource(): _resolve_moe_mesh_resource(MeshResource(fsdp_resource="tp", ep_resource=("ep", "tp"))) +@pytest.mark.parametrize( + "resource,outer_axes", + [ + (MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep"), ("dp", "fsdp")), + ( + MeshResource(dp_resource="dp", fsdp_resource=("fsdp", "ep"), ep_resource="ep"), + ("dp", "fsdp"), + ), + (MeshResource(fsdp_resource=("ep", "fsdp", "tp"), ep_resource=("ep", "tp")), ("fsdp",)), + ], +) +def test_ep_outer_axes_flatten_tuple_resources(resource, outer_axes): + """EP groups count each non-EP axis of a tuple DP or FSDP resource once.""" + from transformer_engine.jax.cpp_extensions import ep + + with global_shard_guard(resource): + assert ep._capture_ep_resource_axes()[1] == outer_axes + + @pytest.mark.parametrize("fsdp_resource", ["fsdp", ("fsdp",), ("fsdp", "ep"), ("ep", "fsdp")]) def test_ep_partitioning_retains_bind_time_resources(fsdp_resource): from transformer_engine.jax.cpp_extensions import ep diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 3d93ecb6a1..1d0b3ed4e0 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -205,7 +205,7 @@ def ep_bootstrap( outer_axes = tuple( dict.fromkeys( a - for a in (resource.dp_resource, resource.fsdp_resource) + for a in (resource.dp_resource, *tex.ep._fsdp_outer_axes(resource)) if a is not None and a not in ep_axes ) ) From 1d7268222818d17df0c7a6d3d111d3811e1caef1 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 16:40:59 -0800 Subject: [PATCH 06/15] Run the TE EP MoE suite on a dp x fsdp x expert mesh Signed-off-by: Teddy Do --- qa/L0_jax_distributed_unittest/test.sh | 3 ++ tests/jax/run_te_ep_moe.sh | 4 +-- tests/jax/test_te_ep_moe.py | 46 ++++++++++++++++++-------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/qa/L0_jax_distributed_unittest/test.sh b/qa/L0_jax_distributed_unittest/test.sh index a563e6908d..bfe19e1616 100644 --- a/qa/L0_jax_distributed_unittest/test.sh +++ b/qa/L0_jax_distributed_unittest/test.sh @@ -47,6 +47,9 @@ wait # >=4 visible GPUs. TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \ || test_fail "test_te_ep_moe.py" +# Same suite on a (dp, fsdp, expert) mesh: dp=2 x fsdp=2 x expert=2 on 8 GPUs. +TE_EP_MOE_DP_SIZE=2 TE_EP_MOE_TP_SIZE=1 TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \ + || test_fail "test_te_ep_moe.py (dp x fsdp x expert)" # Exercise the multi-GPU tutorial in docs/examples/jax (needs >= 4 GPUs; # auto-skips otherwise). CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -m pytest -c $TE_PATH/tests/jax/pytest.ini -v --junitxml=$XML_LOG_DIR/pytest_docs_examples_jax_distributed.xml -k multi_gpu $TE_PATH/docs/examples/jax/ || test_fail "docs/examples/jax (multi-GPU)" diff --git a/tests/jax/run_te_ep_moe.sh b/tests/jax/run_te_ep_moe.sh index e3c081831c..bc52db9a1b 100755 --- a/tests/jax/run_te_ep_moe.sh +++ b/tests/jax/run_te_ep_moe.sh @@ -8,7 +8,7 @@ # its own --num-process=N --process-id=i, and waits for all of them. Each # child calls jax.distributed.initialize(..., local_device_ids=process_id) # so each Python process only sees its one GPU as a local device and the -# participating processes form a global (fsdp, expert[, tensor]) mesh. +# participating processes form a global ([dp,] fsdp, expert[, tensor]) mesh. set -euo pipefail @@ -38,7 +38,7 @@ echo "============================================================" echo "TE-EP MoE MULTIPROCESS test (one process per GPU, ${NUM_GPUS} GPUs)" echo " test file : $TEST_FILE" echo " coordinator : $TE_EP_MOE_COORDINATOR_ADDRESS" -echo " TP size / input rank: ${TE_EP_MOE_TP_SIZE:-2} / ${TE_EP_MOE_INPUT_RANK:-auto}" +echo " DP / TP size / input rank: ${TE_EP_MOE_DP_SIZE:-1} / ${TE_EP_MOE_TP_SIZE:-2} / ${TE_EP_MOE_INPUT_RANK:-auto}" echo " XLA_PYTHON_CLIENT_PREALLOCATE: $XLA_PYTHON_CLIENT_PREALLOCATE" echo " XLA_PYTHON_CLIENT_MEM_FRACTION: $XLA_PYTHON_CLIENT_MEM_FRACTION" if [ "$#" -gt 0 ]; then diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 2e7db3f733..7c1e91150d 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -7,7 +7,7 @@ The launcher ``tests/jax/run_te_ep_moe.sh`` forks one pytest process per visible GPU. Each process binds to exactly one device via ``jax.distributed.initialize(..., local_device_ids=process_id)``; the -participating processes form a global ``(fsdp, expert, tensor)`` mesh through JAX's +participating processes form a global ``([dp,] fsdp, expert[, tensor])`` mesh through JAX's distributed runtime. How to run @@ -140,25 +140,39 @@ def _read_mp_options(): EXPERT_AXIS = "expert" TENSOR_AXIS = "tensor" FSDP_AXIS = "fsdp" +DP_AXIS = "dp" EP_SIZE = 2 TP_SIZE = int(os.environ.get("TE_EP_MOE_TP_SIZE", "2")) +# DP_SIZE > 1 adds an outer ``dp`` axis, so EP groups must count dp x fsdp. +DP_SIZE = int(os.environ.get("TE_EP_MOE_DP_SIZE", "1")) INPUT_RANK = int(os.environ.get("TE_EP_MOE_INPUT_RANK", "2" if TP_SIZE > 1 else "3")) -FSDP_SIZE = jax.device_count() // (EP_SIZE * TP_SIZE) +FSDP_SIZE = jax.device_count() // (DP_SIZE * EP_SIZE * TP_SIZE) assert TP_SIZE in (1, 2), f"TE_EP_MOE_TP_SIZE must be 1 or 2, got {TP_SIZE}" assert INPUT_RANK in (2, 3), f"TE_EP_MOE_INPUT_RANK must be 2 or 3, got {INPUT_RANK}" EP_AXIS = EXPERT_AXIS if TP_SIZE == 1 else (EXPERT_AXIS, TENSOR_AXIS) EP_AXES = (EP_AXIS,) if isinstance(EP_AXIS, str) else EP_AXIS +DP_RESOURCE = DP_AXIS if DP_SIZE > 1 else None +OUTER_AXES = (FSDP_AXIS,) if DP_SIZE == 1 else (DP_AXIS, FSDP_AXIS) # With TP, the MoE block overrides the global resource so EP spans (expert, tensor). EXPLICIT_MESH_RESOURCE = ( - None if TP_SIZE == 1 else MeshResource(ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) + None + if TP_SIZE == 1 + else MeshResource(dp_resource=DP_RESOURCE, ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) ) -MESH_SHAPE = (FSDP_SIZE, EP_SIZE) if TP_SIZE == 1 else (FSDP_SIZE, EP_SIZE, TP_SIZE) -MESH_AXES = (FSDP_AXIS, *EP_AXES) +OUTER_SHAPE = (FSDP_SIZE,) if DP_SIZE == 1 else (DP_SIZE, FSDP_SIZE) +MESH_SHAPE = (*OUTER_SHAPE, EP_SIZE) if TP_SIZE == 1 else (*OUTER_SHAPE, EP_SIZE, TP_SIZE) +MESH_AXES = (*OUTER_AXES, *EP_AXES) COMBINED_EP_SIZE = EP_SIZE * TP_SIZE -NUM_DEVICES_REQUIRED = FSDP_SIZE * EP_SIZE * TP_SIZE +NUM_DEVICES_REQUIRED = DP_SIZE * FSDP_SIZE * EP_SIZE * TP_SIZE +if FSDP_SIZE == 0: + pytest.skip( + f"dp={DP_SIZE} x expert={EP_SIZE} x tensor={TP_SIZE} needs more than" + f" {jax.device_count()} devices", + allow_module_level=True, + ) assert jax.device_count() == NUM_DEVICES_REQUIRED, ( f"device_count {jax.device_count()} must equal {NUM_DEVICES_REQUIRED} for " - f"expert={EP_SIZE} x fsdp={FSDP_SIZE} x tensor={TP_SIZE}" + f"dp={DP_SIZE} x expert={EP_SIZE} x fsdp={FSDP_SIZE} x tensor={TP_SIZE}" ) LOGICAL_AXIS_RULES = ( @@ -167,7 +181,7 @@ def _read_mp_options(): ("gated_weight_fsdp", FSDP_AXIS), ("embed", FSDP_AXIS), ("mlp", None), - ("batch", (FSDP_AXIS, *EP_AXES)), + ("batch", (*OUTER_AXES, *EP_AXES)), ) # Small shapes so the parity tests stay tight on bf16. The block still @@ -217,12 +231,12 @@ def use_cudnn_fusion(request): def mesh(use_cudnn_fusion): if jax.device_count() < NUM_DEVICES_REQUIRED: pytest.skip( - f"Need {NUM_DEVICES_REQUIRED} devices for expert={EP_SIZE} x " + f"Need {NUM_DEVICES_REQUIRED} devices for dp={DP_SIZE} x expert={EP_SIZE} x " f"fsdp={FSDP_SIZE} x tensor={TP_SIZE};" f" have {jax.device_count()}" ) # Both ``expert`` and ``tensor`` act as EP axes. Keep the compound EP - # axes innermost so each FSDP replica owns one consecutive NCCL EP group. + # axes innermost so each DP/FSDP replica owns one consecutive NCCL EP group. devices = mesh_utils.create_device_mesh(MESH_SHAPE) mesh_obj = Mesh(devices, axis_names=MESH_AXES) @@ -243,7 +257,7 @@ def mesh(use_cudnn_fusion): # and cannot run from inside jax.jit. Sized to the worst-case recv_pr # across _CONFIGS so every parametrized config is bootstrap-compatible. with mesh_obj, global_shard_guard( - MeshResource(ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS) + MeshResource(dp_resource=DP_RESOURCE, ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS) ): ep_bootstrap( world_size=num_procs, @@ -470,7 +484,7 @@ def _strong_expert_bias_init(key, shape, dtype): def _shard_inputs(x, mesh): # Match the layout moe.py re-pins to: outer dp axes, then ep innermost. - spec = P((FSDP_AXIS, *EP_AXES), *([None] * (x.ndim - 1))) + spec = P((*OUTER_AXES, *EP_AXES), *([None] * (x.ndim - 1))) return jax.lax.with_sharding_constraint(x, NamedSharding(mesh, spec)) @@ -481,7 +495,9 @@ class _Combo: def __enter__(self_inner): self_inner._m = mesh.__enter__() self_inner._gs = global_shard_guard( - MeshResource(ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS) + MeshResource( + dp_resource=DP_RESOURCE, ep_resource=EXPERT_AXIS, fsdp_resource=FSDP_AXIS + ) ) self_inner._gs.__enter__() self_inner._ar = nn_partitioning.axis_rules(LOGICAL_AXIS_RULES) @@ -725,7 +741,9 @@ def test_mesh_resource_api_forward_and_backward(mesh, quant_before_fsdp_ag, make quantization_recipe=MXFP8BlockScaling(), quant_before_fsdp_ag=quant_before_fsdp_ag ) candidate = baseline.clone( - mesh_resource=MeshResource(ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) + mesh_resource=MeshResource( + dp_resource=DP_RESOURCE, ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS + ) ) x = _make_inputs(jax.random.PRNGKey(51)) variables, baseline_out, _, _ = _init_apply(baseline, mesh, x, jax.random.PRNGKey(52)) From 134489f4fc511cfef3c84e0dd523e65aeacce4ac Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 18:13:56 -0800 Subject: [PATCH 07/15] Drop ep_axes from the standalone EP ops and resolve EP from MeshResource Signed-off-by: Teddy Do --- tests/jax/test_multi_process_ep.py | 10 +- transformer_engine/jax/cpp_extensions/ep.py | 110 ++++++-------------- transformer_engine/jax/ep.py | 50 +++------ 3 files changed, 52 insertions(+), 118 deletions(-) diff --git a/tests/jax/test_multi_process_ep.py b/tests/jax/test_multi_process_ep.py index a0e7a6a5a1..ec6084b642 100644 --- a/tests/jax/test_multi_process_ep.py +++ b/tests/jax/test_multi_process_ep.py @@ -1437,15 +1437,13 @@ def test_compound_ep_axis(self): domains.setdefault(root, {})[col] = rank self.assertEqual([domains[0][i] for i in range(4)], [0, 1, 2, 3]) - with mesh: - ep_axes = ("expert", "tensor") - ok, ep_axis, outer_axes = _leading_axis_ok( - PartitionSpec(ep_axes, None, None), ep_axes=ep_axes - ) + ep_axes = ("expert", "tensor") + with mesh, global_shard_guard(MeshResource(ep_resource=ep_axes)): + ok, ep_axis, outer_axes = _leading_axis_ok(PartitionSpec(ep_axes, None, None)) self.assertTrue(ok) self.assertEqual(ep_axis, ("expert", "tensor")) self.assertEqual(outer_axes, ()) - self.assertTrue(_ep_spec_ok(PartitionSpec(ep_axes, None, None), 2, ep_axes=ep_axes)) + self.assertTrue(_ep_spec_ok(PartitionSpec(ep_axes, None, None), 2)) def test_ep_flattened_replica_groups_multi_domain(self): """Verify flattened replica IDs are grouped by DP coordinate.""" diff --git a/transformer_engine/jax/cpp_extensions/ep.py b/transformer_engine/jax/cpp_extensions/ep.py index 2d0acc440d..0044fcfc44 100644 --- a/transformer_engine/jax/cpp_extensions/ep.py +++ b/transformer_engine/jax/cpp_extensions/ep.py @@ -256,13 +256,12 @@ def _ep_axis_entry(ep_axes: Tuple[str, ...]): return ep_axes[0] if len(ep_axes) == 1 else ep_axes -def _resolve_ep_axes(ep_axes=None) -> Tuple[str, ...]: - """Resolve explicit EP axes, falling back to MeshResource for compatibility.""" - if ep_axes is None: - ep_axes = global_mesh_resource().ep_resource - if ep_axes is None: - raise ValueError("EP mesh axes must be passed explicitly or set as ep_resource.") - return _normalize_ep_axes(ep_axes) +def _resolve_ep_axes(resource=None) -> Tuple[str, ...]: + """Resolve the EP axes of ``resource``, falling back to the active MeshResource.""" + resource = global_mesh_resource() if resource is None else resource + if resource.ep_resource is None: + raise ValueError("EP mesh axes must be set as MeshResource.ep_resource.") + return _normalize_ep_axes(resource.ep_resource) def _fsdp_outer_axes(resource): @@ -274,27 +273,22 @@ def _fsdp_outer_axes(resource): return () if fsdp_resource is None else (fsdp_resource,) -def _capture_ep_resource_axes(ep_axes=None): +def _capture_ep_resource_axes(): """Capture physical axes at bind time for asynchronous SPMD partitioning.""" - try: - resource = global_mesh_resource() - except AssertionError: - if ep_axes is None: - raise - return _ep_axis_entry(_normalize_ep_axes(ep_axes)), () + resource = global_mesh_resource() + ep_axes = _resolve_ep_axes(resource) outer_axes = tuple( dict.fromkeys( axis for axis in (resource.dp_resource, *_fsdp_outer_axes(resource)) if axis is not None ) ) - ep_axes = _resolve_ep_axes(ep_axes) return _ep_axis_entry(ep_axes), tuple(a for a in outer_axes if a not in ep_axes) -def _leading_axis_ok(spec, resource_axes=None, mesh=None, *, ep_axes=None): +def _leading_axis_ok(spec, resource_axes=None, mesh=None): """Validate replicated trailing dimensions and EP plus optional outer axes.""" ep_axis, outer_axes = ( - resource_axes if resource_axes is not None else _capture_ep_resource_axes(ep_axes) + resource_axes if resource_axes is not None else _capture_ep_resource_axes() ) if len(spec) < 2 or ep_axis is None: return False, ep_axis, outer_axes @@ -313,10 +307,10 @@ def _leading_axis_ok(spec, resource_axes=None, mesh=None, *, ep_axes=None): return all(a in allowed for a in elts), ep_axis, outer_axes -def _ep_outer_axis(ep_axes=None): +def _ep_outer_axis(): """Legacy single outer axis used by the standalone EP custom VJP wrapper.""" resource = global_mesh_resource() - ep_axes = _resolve_ep_axes(ep_axes) + ep_axes = _resolve_ep_axes(resource) dp_resource = None if resource.dp_resource in ep_axes else resource.dp_resource fsdp_axes = tuple(axis for axis in _fsdp_outer_axes(resource) if axis not in ep_axes) fsdp_resource = fsdp_axes[0] if len(fsdp_axes) == 1 else fsdp_axes or None @@ -345,10 +339,10 @@ def _ep_output_spec(*trailing, resource_axes=None): return PartitionSpec(leading, *trailing) -def _ep_spec_ok(spec, trailing_count, resource_axes=None, *, ep_axes=None): +def _ep_spec_ok(spec, trailing_count, resource_axes=None): """Validate EP-output axes, allowing JAX to drop size-one mesh axes.""" ep_axis, outer_axes = ( - resource_axes if resource_axes is not None else _capture_ep_resource_axes(ep_axes) + resource_axes if resource_axes is not None else _capture_ep_resource_axes() ) if len(spec) != 1 + trailing_count: return False @@ -1439,7 +1433,7 @@ def shardy_sharding_rule(*args): @_on_collective_stream -def _ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): +def _ep_prepare(cfg: EpLayerConfig, topk_idx): """Exchange routing metadata for ``cfg``; return ``(token_counts, total_recv_tokens, handle_mem)``. ``total_recv_tokens`` is the per-rank pre-drop recv-slot total (includes tokens dropped on overflow).""" @@ -1453,7 +1447,7 @@ def _ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): dispatch_output_per_expert_alignment=int(cfg.dispatch_output_per_expert_alignment), topk_idx_dtype=int(cfg.topk_idx_dtype), is_outer=True, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) @@ -1466,7 +1460,6 @@ def _ep_dispatch_fwd( tokens_scale_inv, topk_weights, recv_capacity_per_rank, - ep_axes=None, ): """Scatter tokens and weights to expert ranks; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, handle_mem)``.""" @@ -1481,11 +1474,11 @@ def _ep_dispatch_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) -def ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): +def ep_prepare(cfg: EpLayerConfig, topk_idx): """Deprecated: use ``ep_prepare_and_dispatch_fwd``. Planned for removal in January 2027.""" warnings.warn( "ep_prepare is deprecated and planned to be removed in January 2027. Use" @@ -1493,17 +1486,11 @@ def ep_prepare(cfg: EpLayerConfig, topk_idx, ep_axes=None): DeprecationWarning, stacklevel=2, ) - return _ep_prepare(cfg, topk_idx, ep_axes=ep_axes) + return _ep_prepare(cfg, topk_idx) def ep_dispatch_fwd( - cfg: EpLayerConfig, - handle_mem, - topk_idx, - tokens, - topk_weights, - recv_capacity_per_rank, - ep_axes=None, + cfg: EpLayerConfig, handle_mem, topk_idx, tokens, topk_weights, recv_capacity_per_rank ): """Deprecated: use ``ep_prepare_and_dispatch_fwd``. Planned for removal in January 2027.""" warnings.warn( @@ -1520,7 +1507,6 @@ def ep_dispatch_fwd( empty_token_scale(tokens), topk_weights, recv_capacity_per_rank, - ep_axes=ep_axes, ) return recv_tokens, recv_topk_weights, handle_mem @@ -1542,13 +1528,7 @@ def _fused_prepare_dispatch_enabled(): @_on_collective_stream def _ep_fused_prepare_and_dispatch_fwd( - cfg: EpLayerConfig, - topk_idx, - tokens, - tokens_scale_inv, - topk_weights, - recv_capacity_per_rank, - ep_axes=None, + cfg: EpLayerConfig, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank ): """Fused prepare + dispatch; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, token_counts, total_recv_tokens, @@ -1563,7 +1543,7 @@ def _ep_fused_prepare_and_dispatch_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) @@ -1573,7 +1553,6 @@ def ep_prepare_and_dispatch_fwd( tokens: Union[jnp.ndarray, ScaledTensor1x], topk_weights, recv_capacity_per_rank, - ep_axes=None, ): """Prepare + dispatch; returns ``(recv_tokens, recv_topk_weights, token_counts, total_recv_tokens, handle_mem)``. @@ -1589,13 +1568,7 @@ def ep_prepare_and_dispatch_fwd( else: tokens_scale_inv = empty_token_scale(tokens) outputs = _ep_prepare_and_dispatch_fwd( - cfg, - topk_idx, - tokens, - tokens_scale_inv, - topk_weights, - recv_capacity_per_rank, - ep_axes=ep_axes, + cfg, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank ) recv_tokens, recv_scale_inv, *rest = outputs if isinstance(tokens_in, ScaledTensor1x): @@ -1612,13 +1585,7 @@ def ep_prepare_and_dispatch_fwd( def _ep_prepare_and_dispatch_fwd( - cfg: EpLayerConfig, - topk_idx, - tokens, - tokens_scale_inv, - topk_weights, - recv_capacity_per_rank, - ep_axes=None, + cfg: EpLayerConfig, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank ): """Prepare + dispatch on raw arrays; returns ``(recv_tokens, recv_scale_inv, recv_topk_weights, token_counts, total_recv_tokens, @@ -1628,15 +1595,9 @@ def _ep_prepare_and_dispatch_fwd( """ if _fused_prepare_dispatch_enabled(): return _ep_fused_prepare_and_dispatch_fwd( - cfg, - topk_idx, - tokens, - tokens_scale_inv, - topk_weights, - recv_capacity_per_rank, - ep_axes=ep_axes, + cfg, topk_idx, tokens, tokens_scale_inv, topk_weights, recv_capacity_per_rank ) - token_counts, total_recv_tokens, handle_mem = _ep_prepare(cfg, topk_idx, ep_axes=ep_axes) + token_counts, total_recv_tokens, handle_mem = _ep_prepare(cfg, topk_idx) recv_tokens, recv_scale_inv, recv_topk_weights, handle_mem = _ep_dispatch_fwd( cfg, handle_mem, @@ -1645,7 +1606,6 @@ def _ep_prepare_and_dispatch_fwd( tokens_scale_inv, topk_weights, recv_capacity_per_rank, - ep_axes=ep_axes, ) return ( recv_tokens, @@ -1659,12 +1619,7 @@ def _ep_prepare_and_dispatch_fwd( @_on_collective_stream def ep_combine_fwd( - cfg: EpLayerConfig, - handle_mem, - expert_out, - num_local_tokens, - out_partition_spec=None, - ep_axes=None, + cfg: EpLayerConfig, handle_mem, expert_out, num_local_tokens, out_partition_spec=None ): """Gather expert outputs back to home ranks. expert_out is pre-weighted.""" out_leading = _normalize_leading_shape(num_local_tokens) @@ -1676,7 +1631,7 @@ def ep_combine_fwd( topk_idx_dtype=int(cfg.topk_idx_dtype), out_leading_shape=out_leading, out_partition_spec=out_partition_spec, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) @@ -1688,7 +1643,6 @@ def ep_dispatch_bwd( g_recv_topk_weights, num_local_tokens, out_partition_spec=None, - ep_axes=None, ): """Backward of dispatch; returns (grad_tokens, grad_topk_weights).""" out_leading = _normalize_leading_shape(num_local_tokens) @@ -1701,12 +1655,12 @@ def ep_dispatch_bwd( topk_idx_dtype=int(cfg.topk_idx_dtype), out_leading_shape=out_leading, out_partition_spec=out_partition_spec, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) @_on_collective_stream -def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank, ep_axes=None): +def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank): """Backward of combine; returns grad_expert_out [num_procs, recv_capacity_per_rank, H].""" return EpCombineBwdPrimitive.outer_primitive.bind( handle_mem, @@ -1716,5 +1670,5 @@ def ep_combine_bwd(cfg: EpLayerConfig, handle_mem, grad, recv_capacity_per_rank, topk_idx_dtype=int(cfg.topk_idx_dtype), recv_capacity_per_rank=recv_capacity_per_rank, is_outer=True, - resource_axes=_capture_ep_resource_axes(ep_axes), + resource_axes=_capture_ep_resource_axes(), ) diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 1d0b3ed4e0..98d74f99be 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -311,11 +311,11 @@ def ep_finalize(): tex.ep.reset_ep_config() -def _default_out_partition_spec(ep_axes=None): +def _default_out_partition_spec(): """Leading-axis default: ``(("dp","ep"),)`` if DP/FSDP is set, else ``("ep",)``.""" - outer = _ep_outer_axis(ep_axes) + outer = _ep_outer_axis() outer_axes = (outer,) if isinstance(outer, str) else outer or () - leading_axes = (*outer_axes, *tex.ep._resolve_ep_axes(ep_axes)) + leading_axes = (*outer_axes, *tex.ep._resolve_ep_axes()) leading = leading_axes[0] if len(leading_axes) == 1 else leading_axes return (leading,) @@ -323,8 +323,8 @@ def _default_out_partition_spec(ep_axes=None): # ── ep_dispatch (custom_vjp) ───────────────────────────────────────────────── -@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5)) -def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=None): +@partial(jax.custom_vjp, nondiff_argnums=(0, 4)) +def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank): """Scatter tokens and weights to expert ranks. ``cfg`` is a per-layer ``EpLayerConfig``; distinct layers may share a @@ -339,33 +339,28 @@ def ep_dispatch(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_ ``drop_on_overflow`` is set. When ``recv_capacity_per_rank`` is not sized for the worst case, detect overflow by ``process_allgather``-ing it, then ``max(...) > recv_capacity_per_rank`` flags an overflowing step. - - ``ep_axes`` may be a mesh axis name or an ordered tuple of names. When it - is ``None``, the active ``MeshResource.ep_resource`` is used. """ - return _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes)[0] + return _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank)[0] -def _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=None): +def _dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank): if not jnp.issubdtype(topk_weights.dtype, jnp.floating): raise TypeError( f"ep_dispatch: topk_weights must be a floating dtype; got {topk_weights.dtype}." ) recv_tokens, recv_topk_weights, token_counts, total_recv_tokens, handle_mem = ( - tex.ep_prepare_and_dispatch_fwd( - cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank, ep_axes=ep_axes - ) + tex.ep_prepare_and_dispatch_fwd(cfg, topk_idx, tokens, topk_weights, recv_capacity_per_rank) ) out_leading = tuple(tokens.shape[:-1]) primal = (recv_tokens, recv_topk_weights, handle_mem, token_counts, total_recv_tokens) return primal, (handle_mem, out_leading) -def _dispatch_bwd(cfg, recv_capacity_per_rank, ep_axes, res, g_outputs): +def _dispatch_bwd(cfg, recv_capacity_per_rank, res, g_outputs): del recv_capacity_per_rank handle_mem, out_leading = res # Re-pin cotangent: XLA transpose can drop the EP axis and feed the FFI a global tensor. - out_spec = _default_out_partition_spec(ep_axes) + out_spec = _default_out_partition_spec() spec = jax.sharding.PartitionSpec(*out_spec) g_recv_tokens = with_sharding_constraint(g_outputs[0], spec) g_recv_topk_weights = with_sharding_constraint(g_outputs[1], spec) @@ -376,7 +371,6 @@ def _dispatch_bwd(cfg, recv_capacity_per_rank, ep_axes, res, g_outputs): g_recv_topk_weights, out_leading, out_partition_spec=out_spec, - ep_axes=ep_axes, ) return (None, grad_tokens, grad_topk_weights) @@ -387,7 +381,7 @@ def _dispatch_bwd(cfg, recv_capacity_per_rank, ep_axes, res, g_outputs): # ── ep_combine (custom_vjp) ────────────────────────────────────────────────── -@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5, 6)) +@partial(jax.custom_vjp, nondiff_argnums=(0, 4, 5)) def ep_combine( cfg, handle_mem, @@ -395,7 +389,6 @@ def ep_combine( expert_out, num_local_tokens, out_sharding=None, - ep_axes=None, ): """Scatter-sum expert outputs back to source ranks. **Unweighted.** @@ -404,8 +397,6 @@ def ep_combine( not through this op. ``num_local_tokens`` is STATIC: int -> ``[T, H]``, tuple -> ``[*tuple, H]``. ``out_sharding`` defaults via ``_default_out_partition_spec``; only the leading dim may be sharded. - ``ep_axes`` accepts an explicit mesh axis name or tuple and otherwise falls - back to ``MeshResource.ep_resource``. """ return _combine_fwd( cfg, @@ -414,7 +405,6 @@ def ep_combine( expert_out, num_local_tokens, out_sharding, - ep_axes, )[0] @@ -425,32 +415,24 @@ def _combine_fwd( expert_out, num_local_tokens, out_sharding, - ep_axes=None, ): del token_counts if out_sharding is None: - out_sharding = _default_out_partition_spec(ep_axes) + out_sharding = _default_out_partition_spec() result = tex.ep_combine_fwd( - cfg, - handle_mem, - expert_out, - num_local_tokens, - out_partition_spec=out_sharding, - ep_axes=ep_axes, + cfg, handle_mem, expert_out, num_local_tokens, out_partition_spec=out_sharding ) return result, (handle_mem, expert_out.shape[-2]) -def _combine_bwd(cfg, _num_local_tokens, _out_sharding, ep_axes, res, g_result): +def _combine_bwd(cfg, _num_local_tokens, _out_sharding, res, g_result): handle_mem, recv_capacity_per_rank = res # Re-pin cotangent (same XLA-transpose workaround as _dispatch_bwd). if _out_sharding is None: - _out_sharding = _default_out_partition_spec(ep_axes) + _out_sharding = _default_out_partition_spec() spec = jax.sharding.PartitionSpec(*_out_sharding) g_result = with_sharding_constraint(g_result, spec) - grad_expert_out = tex.ep_combine_bwd( - cfg, handle_mem, g_result, recv_capacity_per_rank, ep_axes=ep_axes - ) + grad_expert_out = tex.ep_combine_bwd(cfg, handle_mem, g_result, recv_capacity_per_rank) return (None, None, grad_expert_out) From 795cdbb2b0d941434a64428c545317d9b67dd523 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 18:16:06 -0800 Subject: [PATCH 08/15] Use _fsdp_outer_axes for EP outer axes in bind-time specs and bootstrap Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 29 +++++++++++---------- transformer_engine/jax/cpp_extensions/ep.py | 13 ++++----- transformer_engine/jax/ep.py | 8 +----- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index a8547e684b..30bfee0698 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -47,15 +47,6 @@ def test_explicit_resource_overrides_global_and_is_snapshotted(): assert quantize is True -def test_global_default_and_duplicate_outer_axis(): - with global_shard_guard( - MeshResource(dp_resource="fsdp", fsdp_resource="fsdp", ep_resource="ep") - ): - resource, quantize = _resolve_moe_mesh_resource() - assert _moe_mesh_axes(resource) == ("ep", ("fsdp",)) - assert quantize is False - - def test_tuple_ep_resource(): resource, _ = _resolve_moe_mesh_resource( MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource=("ep", "tp")) @@ -68,20 +59,30 @@ def test_tuple_ep_resource(): @pytest.mark.parametrize( "resource,outer_axes", [ - (MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep"), ("dp", "fsdp")), ( - MeshResource(dp_resource="dp", fsdp_resource=("fsdp", "ep"), ep_resource="ep"), + MeshResource( + dp_resource="dp", fsdp_resource=("ep", "fsdp", "tp"), ep_resource=("ep", "tp") + ), ("dp", "fsdp"), ), - (MeshResource(fsdp_resource=("ep", "fsdp", "tp"), ep_resource=("ep", "tp")), ("fsdp",)), + (MeshResource(dp_resource="tp", fsdp_resource="ep", ep_resource=("ep", "tp")), ()), ], ) def test_ep_outer_axes_flatten_tuple_resources(resource, outer_axes): - """EP groups count each non-EP axis of a tuple DP or FSDP resource once.""" + """EP outer axes flatten a tuple FSDP resource and drop every axis of a tuple EP.""" from transformer_engine.jax.cpp_extensions import ep with global_shard_guard(resource): - assert ep._capture_ep_resource_axes()[1] == outer_axes + assert ep._capture_ep_resource_axes() == (("ep", "tp"), outer_axes) + + +def test_global_default_and_duplicate_outer_axis(): + with global_shard_guard( + MeshResource(dp_resource="fsdp", fsdp_resource="fsdp", ep_resource="ep") + ): + resource, quantize = _resolve_moe_mesh_resource() + assert _moe_mesh_axes(resource) == ("ep", ("fsdp",)) + assert quantize is False @pytest.mark.parametrize("fsdp_resource", ["fsdp", ("fsdp",), ("fsdp", "ep"), ("ep", "fsdp")]) diff --git a/transformer_engine/jax/cpp_extensions/ep.py b/transformer_engine/jax/cpp_extensions/ep.py index 0044fcfc44..f62ed49cc4 100644 --- a/transformer_engine/jax/cpp_extensions/ep.py +++ b/transformer_engine/jax/cpp_extensions/ep.py @@ -267,15 +267,16 @@ def _resolve_ep_axes(resource=None) -> Tuple[str, ...]: def _fsdp_outer_axes(resource): """FSDP axes of ``resource`` that count as EP outer axes, as a tuple.""" fsdp_resource = resource.fsdp_resource + ep_axes = _resolve_ep_axes(resource) if isinstance(fsdp_resource, tuple): - # A dense FSDP tuple may include the EP axis, which never counts as an outer axis. - return tuple(axis for axis in fsdp_resource if axis != resource.ep_resource) - return () if fsdp_resource is None else (fsdp_resource,) + # A dense FSDP tuple may include EP axes, which never count as outer axes. + return tuple(axis for axis in fsdp_resource if axis not in ep_axes) + return () if fsdp_resource is None or fsdp_resource in ep_axes else (fsdp_resource,) -def _capture_ep_resource_axes(): +def _capture_ep_resource_axes(resource=None): """Capture physical axes at bind time for asynchronous SPMD partitioning.""" - resource = global_mesh_resource() + resource = global_mesh_resource() if resource is None else resource ep_axes = _resolve_ep_axes(resource) outer_axes = tuple( dict.fromkeys( @@ -312,7 +313,7 @@ def _ep_outer_axis(): resource = global_mesh_resource() ep_axes = _resolve_ep_axes(resource) dp_resource = None if resource.dp_resource in ep_axes else resource.dp_resource - fsdp_axes = tuple(axis for axis in _fsdp_outer_axes(resource) if axis not in ep_axes) + fsdp_axes = _fsdp_outer_axes(resource) fsdp_resource = fsdp_axes[0] if len(fsdp_axes) == 1 else fsdp_axes or None if dp_resource is not None and get_mesh_axis_size(dp_resource) > 1: return dp_resource diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 98d74f99be..4a19d382d3 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -202,13 +202,7 @@ def ep_bootstrap( ep_size = int(np.prod([mesh.shape[axis] for axis in ep_axes])) # num_ep_groups counts only the distinct-token (dp/fsdp) axes; replicated # axes (tp, pp, ...) do not create distinct EP-output slabs. - outer_axes = tuple( - dict.fromkeys( - a - for a in (resource.dp_resource, *tex.ep._fsdp_outer_axes(resource)) - if a is not None and a not in ep_axes - ) - ) + _, outer_axes = tex.ep._capture_ep_resource_axes(resource) num_ep_groups = int(np.prod([get_mesh_axis_size(a) for a in outer_axes])) if num_experts % ep_size != 0: raise ValueError(f"num_experts ({num_experts}) must be divisible by ep_size ({ep_size}).") From 27cc7f12a8ce5123824bf50d0b371d1cb3812dce Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 18:16:55 -0800 Subject: [PATCH 09/15] Test tuple FSDP and EP resources in the default EP out spec Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 30bfee0698..621bc72c44 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -76,6 +76,21 @@ def test_ep_outer_axes_flatten_tuple_resources(resource, outer_axes): assert ep._capture_ep_resource_axes() == (("ep", "tp"), outer_axes) +def test_tuple_resources_flatten_in_default_spec_and_sizes(monkeypatch): + """Tuple FSDP and EP resources give a flat default EP output spec and product sizes.""" + from types import SimpleNamespace + from transformer_engine.jax import sharding + from transformer_engine.jax.ep import _default_out_partition_spec + + mesh = SimpleNamespace(shape={"fsdp": 2, "data": 2, "ep": 2, "tp": 2}, empty=False) + monkeypatch.setattr(sharding, "_get_mesh", lambda: mesh) + resource = MeshResource(fsdp_resource=("fsdp", "data", "ep"), ep_resource=("ep", "tp")) + with global_shard_guard(resource): + assert _default_out_partition_spec() == (("fsdp", "data", "ep", "tp"),) + assert sharding.ep_axis_size() == 4 + assert sharding.get_active_resource_axis("ep_resource") == ("ep", "tp") + + def test_global_default_and_duplicate_outer_axis(): with global_shard_guard( MeshResource(dp_resource="fsdp", fsdp_resource="fsdp", ep_resource="ep") From d788b99cabe6979650fb2d355c09ad1c0fb11a01 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 18:17:27 -0800 Subject: [PATCH 10/15] Run the second TE EP MoE pass at dp=2 only with 8 or more GPUs Signed-off-by: Teddy Do --- qa/L0_jax_distributed_unittest/test.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qa/L0_jax_distributed_unittest/test.sh b/qa/L0_jax_distributed_unittest/test.sh index bfe19e1616..185a2a95d6 100644 --- a/qa/L0_jax_distributed_unittest/test.sh +++ b/qa/L0_jax_distributed_unittest/test.sh @@ -47,9 +47,14 @@ wait # >=4 visible GPUs. TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \ || test_fail "test_te_ep_moe.py" -# Same suite on a (dp, fsdp, expert) mesh: dp=2 x fsdp=2 x expert=2 on 8 GPUs. -TE_EP_MOE_DP_SIZE=2 TE_EP_MOE_TP_SIZE=1 TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \ - || test_fail "test_te_ep_moe.py (dp x fsdp x expert)" +# Same suite with single-axis EP: dp=2 x fsdp=2 x expert=2 with 8 or more GPUs, +# else fsdp x expert, so that fsdp stays above 1 on either runner. +EP_MOE_DP_SIZE=1 +if [ "${NUM_GPUS:-$(nvidia-smi -L | wc -l)}" -ge 8 ]; then + EP_MOE_DP_SIZE=2 +fi +TE_EP_MOE_DP_SIZE=$EP_MOE_DP_SIZE TE_EP_MOE_TP_SIZE=1 TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \ + || test_fail "test_te_ep_moe.py (TP=1)" # Exercise the multi-GPU tutorial in docs/examples/jax (needs >= 4 GPUs; # auto-skips otherwise). CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -m pytest -c $TE_PATH/tests/jax/pytest.ini -v --junitxml=$XML_LOG_DIR/pytest_docs_examples_jax_distributed.xml -k multi_gpu $TE_PATH/docs/examples/jax/ || test_fail "docs/examples/jax (multi-GPU)" From 8b41cf165a68f5daa08e3132e6cb7f0bf59e5a58 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Mon, 16 Nov 2026 18:17:54 -0800 Subject: [PATCH 11/15] Reject a non-MeshResource mesh_resource in ep_bootstrap Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 4 ++++ transformer_engine/jax/ep.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 621bc72c44..7a44c9c066 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -213,10 +213,14 @@ def test_invalid_resource(resource, quantize, error): def test_bool_and_resource_types(): + from transformer_engine.jax.ep import ep_bootstrap + with pytest.raises(TypeError, match="must be a bool"): _resolve_moe_mesh_resource(MeshResource(ep_resource="ep"), "true") with pytest.raises(TypeError, match="must be a MeshResource"): _resolve_moe_mesh_resource("ep") + with pytest.raises(TypeError, match="must be a MeshResource"): + ep_bootstrap(None, None, 2, 1, 1, 1, mesh_resource="ep") @pytest.mark.parametrize("quantize", [False, True]) diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 4a19d382d3..1cf6874835 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -17,6 +17,7 @@ from transformer_engine.jax.cpp_extensions.ep import _ep_outer_axis, _normalize_ep_axes from transformer_engine.jax.cpp_extensions.misc import jax_dtype_to_te_dtype from transformer_engine.jax.sharding import ( + MeshResource, _get_mesh, get_num_devices_in_mesh, global_mesh_resource, @@ -158,6 +159,8 @@ def ep_bootstrap( mesh_resource: MeshResource naming the EP and outer DP/FSDP mesh axes. ``None`` uses the active global_shard_guard. """ + if mesh_resource is not None and not isinstance(mesh_resource, MeshResource): + raise TypeError("mesh_resource must be a MeshResource or None.") if rank is None: rank = jax.process_index() if jnp.dtype(max_token_dtype) != jnp.bfloat16: From a8a887e78a8947532f25549ead69ffbb0aa93796 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Tue, 17 Nov 2026 16:43:39 -0800 Subject: [PATCH 12/15] Test size-one tuple EP axes in the EP leading-axis check Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 7a44c9c066..08326f37c1 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -133,6 +133,23 @@ def test_ep_leading_axis_canonicalization(ep_size, expected): assert not ep._leading_axis_ok(P("fsdp", None, None), axes, missing_ep_mesh)[0] +@pytest.mark.parametrize("tp_size,expected", [(1, True), (2, False)]) +def test_tuple_ep_leading_axis_canonicalization(tp_size, expected): + """A tuple EP axis may be absent from the spec only when its mesh size is one.""" + from types import SimpleNamespace + from jax.sharding import PartitionSpec as P + from transformer_engine.jax.cpp_extensions import ep + + mesh = SimpleNamespace(shape={"fsdp": 2, "ep": 2, "tp": tp_size}) + axes = (("ep", "tp"), ("fsdp",)) + assert ep._leading_axis_ok(P(("fsdp", "ep"), None, None), axes, mesh)[0] is expected + assert ep._leading_axis_ok(P("ep", None, None), axes, mesh)[0] is expected + assert ep._leading_axis_ok(P(("fsdp", "ep", "tp"), None, None), axes, mesh)[0] + assert not ep._leading_axis_ok(P(("fsdp", "tp"), None, None), axes, mesh)[0] + assert not ep._leading_axis_ok(P(None, None, None), axes, mesh)[0] + assert not ep._leading_axis_ok(P(("fsdp", "ep"), None, None), axes)[0] + + def test_fused_ep_partitioning_retains_bind_time_resources(monkeypatch): from types import SimpleNamespace from jax.sharding import NamedSharding, PartitionSpec as P From 7501b01def5008e946186afaf8301bf0cd6a9c42 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Thu, 19 Nov 2026 12:20:11 -0800 Subject: [PATCH 13/15] Gate the TE EP MoE tests on sm100+ as on main Signed-off-by: Teddy Do --- tests/jax/test_te_ep_moe.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 7c1e91150d..f8d9b95534 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -112,11 +112,12 @@ def _read_mp_options(): from transformer_engine_jax import get_device_compute_capability -# The TE EP NCCL primitives and the grouped-GEMM fallback used by the MoE -# custom_vjp are supported on Hopper (sm_90+) and newer GPUs. -if get_device_compute_capability(0) < 90: +# Grouped GEMM in the MoE custom_vjp requires Blackwell (sm_100+). The +# TE EP NCCL primitives themselves need SM>=90, but the FFN body uses +# grouped_gemm, so the file as a whole gates on sm_100+. +if get_device_compute_capability(0) < 100: pytest.skip( - "MoE TE EP tests require Hopper (sm_90+) or newer", + "MoE TE EP tests require Blackwell (sm_100+) for grouped GEMM", allow_module_level=True, ) @@ -652,7 +653,7 @@ def _quantization_recipe(quantization): ] if ( - get_device_compute_capability(0) >= 90 + get_device_compute_capability(0) >= 100 and os.environ.get("NVTE_JAX_ENFORCE_V2_GROUPED_GEMM", "0") != "1" ): _QUANTIZATION_CASES.append(pytest.param("mxfp8", id="mxfp8")) From c59f64f441074833aa15b9b0c6a2ef76c95eada4 Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Fri, 20 Nov 2026 10:13:14 -0800 Subject: [PATCH 14/15] Use every DP/FSDP outer axis in the default standalone EP layout Signed-off-by: Teddy Do --- tests/jax/moe/test_moe_resource_api.py | 26 ++++++++----- tests/jax/test_te_ep_moe.py | 41 ++++++++++++++++++++- transformer_engine/jax/cpp_extensions/ep.py | 16 +------- transformer_engine/jax/ep.py | 7 ++-- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 08326f37c1..7d0cde661d 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -187,35 +187,41 @@ def bind(*args, **kwargs): "fsdp_resource,outer_axis", [("fsdp", "fsdp"), (("fsdp", "ep"), "fsdp"), (("ep", "fsdp"), "fsdp"), (("ep",), None)], ) -def test_ep_outer_axis_excludes_ep_from_fsdp_tuple(monkeypatch, fsdp_resource, outer_axis): - from types import SimpleNamespace - from transformer_engine.jax import sharding - from transformer_engine.jax.cpp_extensions import ep +def test_default_ep_out_spec_excludes_ep_from_fsdp_tuple(fsdp_resource, outer_axis): from transformer_engine.jax.ep import _default_out_partition_spec - monkeypatch.setattr(sharding, "_get_mesh", lambda: SimpleNamespace(shape={"fsdp": 2, "ep": 4})) with global_shard_guard(MeshResource(fsdp_resource=fsdp_resource, ep_resource="ep")): - assert ep._ep_outer_axis() == outer_axis leading = (outer_axis, "ep") if outer_axis is not None else "ep" assert _default_out_partition_spec() == (leading,) -def test_default_ep_out_spec_flattens_two_outer_fsdp_axes(monkeypatch): +def test_default_ep_out_spec_flattens_two_outer_fsdp_axes(): from types import SimpleNamespace - from transformer_engine.jax import sharding from transformer_engine.jax.cpp_extensions import ep from transformer_engine.jax.ep import _default_out_partition_spec mesh = SimpleNamespace(shape={"fsdp1": 2, "fsdp2": 2, "ep": 2}) - monkeypatch.setattr(sharding, "_get_mesh", lambda: mesh) resource = MeshResource(fsdp_resource=("fsdp1", "fsdp2", "ep"), ep_resource="ep") with global_shard_guard(resource): - assert ep._ep_outer_axis() == ("fsdp1", "fsdp2") spec = _default_out_partition_spec() assert spec == (("fsdp1", "fsdp2", "ep"),) assert ep._leading_per_shard((16, 4), spec[0], mesh) == (2, 4) +def test_default_ep_out_spec_uses_dp_and_fsdp(): + """The default EP out spec shards over dp and fsdp, as ep_bootstrap counts EP groups.""" + from types import SimpleNamespace + from transformer_engine.jax.cpp_extensions import ep + from transformer_engine.jax.ep import _default_out_partition_spec + + mesh = SimpleNamespace(shape={"dp": 2, "fsdp": 2, "ep": 2}) + resource = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep") + with global_shard_guard(resource): + spec = _default_out_partition_spec() + assert spec == (("dp", "fsdp", "ep"),) + assert ep._leading_per_shard((16, 4), spec[0], mesh) == (2, 4) + + @pytest.mark.parametrize( "resource,quantize,error", [ diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index f8d9b95534..2f6393c3e1 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -130,7 +130,13 @@ def _read_mp_options(): record_ep_bootstrap_signature_for_moe, ) from transformer_engine.jax.cpp_extensions.ep import get_ep_config -from transformer_engine.jax.ep import ep_bootstrap, ep_finalize +from transformer_engine.jax.ep import ( + EpLayerConfig, + ep_bootstrap, + ep_combine, + ep_dispatch, + ep_finalize, +) from transformer_engine.common.recipe import Format, MXFP8BlockScaling from transformer_engine.jax.sharding import MeshResource, global_shard_guard @@ -735,6 +741,39 @@ def test_ep_bootstrap_matches_mesh_resource(): assert cfg.num_ep_groups == NUM_DEVICES_REQUIRED // COMBINED_EP_SIZE +def test_standalone_dispatch_combine_default_layout(mesh): + """ep_dispatch and ep_combine with their default layouts round-trip tokens and gradients.""" + num_tokens = 2 * NUM_DEVICES_REQUIRED + tokens = jax.random.normal(jax.random.PRNGKey(61), (num_tokens, HIDDEN), DTYPE) + # Distinct experts per token and weights summing to one make the round trip the identity. + topk_idx = ( + jnp.arange(num_tokens * TOPK, dtype=jnp.int32).reshape(num_tokens, TOPK) % NUM_EXPERTS + ) + topk_weights = jnp.full((num_tokens, TOPK), 1.0 / TOPK, jnp.float32) + cfg = EpLayerConfig(top_k=TOPK, dispatch_output_per_expert_alignment=128) + recv_capacity_per_rank = get_ep_config().recv_capacity_per_rank + + def loss_fn(x, idx, w): + recv_x, recv_w, handle_mem, token_counts, _ = ep_dispatch( + cfg, idx, x, w, recv_capacity_per_rank + ) + recv_x, recv_w = (_shard_inputs(a, mesh) for a in (recv_x, recv_w)) + expert_out = recv_x * recv_w[..., None].astype(recv_x.dtype) + out = ep_combine(cfg, handle_mem, token_counts, expert_out, num_tokens) + return 0.5 * jnp.sum(out.astype(jnp.float32) ** 2), out + + resource = MeshResource(dp_resource=DP_RESOURCE, ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) + with mesh, global_shard_guard(resource): + args = [_shard_inputs(a, mesh) for a in (tokens, topk_idx, topk_weights)] + (_, out), grad = jax.jit(jax.value_and_grad(loss_fn, has_aux=True))(*args) + # The loss is half the squared norm of the tokens, so its gradient is the tokens too. + expected = np.asarray(tokens, np.float32) + for actual in (out, grad): + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), expected, **FWD_TOLERANCE["bf16"] + ) + + @pytest.mark.parametrize("quant_before_fsdp_ag", [False, True]) def test_mesh_resource_api_forward_and_backward(mesh, quant_before_fsdp_ag, make_block): """Explicit and global resources give identical forward and backward results.""" diff --git a/transformer_engine/jax/cpp_extensions/ep.py b/transformer_engine/jax/cpp_extensions/ep.py index f62ed49cc4..78b1307adb 100644 --- a/transformer_engine/jax/cpp_extensions/ep.py +++ b/transformer_engine/jax/cpp_extensions/ep.py @@ -30,7 +30,7 @@ from .base import BasePrimitive, register_primitive from .misc import TEDType, jax_dtype_to_te_dtype from ..quantize.tensor import ScaledTensor1x, ScaledTensorFactory -from ..sharding import global_mesh_resource, get_mesh_axis_size +from ..sharding import global_mesh_resource from ..version_utils import is_collective_stream_supported, is_xla_ffi_collectives_supported @@ -308,20 +308,6 @@ def _leading_axis_ok(spec, resource_axes=None, mesh=None): return all(a in allowed for a in elts), ep_axis, outer_axes -def _ep_outer_axis(): - """Legacy single outer axis used by the standalone EP custom VJP wrapper.""" - resource = global_mesh_resource() - ep_axes = _resolve_ep_axes(resource) - dp_resource = None if resource.dp_resource in ep_axes else resource.dp_resource - fsdp_axes = _fsdp_outer_axes(resource) - fsdp_resource = fsdp_axes[0] if len(fsdp_axes) == 1 else fsdp_axes or None - if dp_resource is not None and get_mesh_axis_size(dp_resource) > 1: - return dp_resource - if fsdp_resource is not None and get_mesh_axis_size(fsdp_resource) > 1: - return fsdp_resource - return dp_resource or fsdp_resource - - def _ep_leading_dims(is_outer): """Leading dim of an EP-output tensor: num_ep_groups*ep_size globally, 1 per shard. Read from EpConfig so abstract-eval needs no active mesh.""" diff --git a/transformer_engine/jax/ep.py b/transformer_engine/jax/ep.py index 1cf6874835..30f6bad37d 100644 --- a/transformer_engine/jax/ep.py +++ b/transformer_engine/jax/ep.py @@ -14,7 +14,7 @@ import transformer_engine_jax import transformer_engine.jax.cpp_extensions as tex -from transformer_engine.jax.cpp_extensions.ep import _ep_outer_axis, _normalize_ep_axes +from transformer_engine.jax.cpp_extensions.ep import _normalize_ep_axes from transformer_engine.jax.cpp_extensions.misc import jax_dtype_to_te_dtype from transformer_engine.jax.sharding import ( MeshResource, @@ -309,9 +309,8 @@ def ep_finalize(): def _default_out_partition_spec(): - """Leading-axis default: ``(("dp","ep"),)`` if DP/FSDP is set, else ``("ep",)``.""" - outer = _ep_outer_axis() - outer_axes = (outer,) if isinstance(outer, str) else outer or () + """Leading-axis default: every DP/FSDP outer axis, then EP, as counted by ``ep_bootstrap``.""" + _, outer_axes = tex.ep._capture_ep_resource_axes() leading_axes = (*outer_axes, *tex.ep._resolve_ep_axes()) leading = leading_axes[0] if len(leading_axes) == 1 else leading_axes return (leading,) From 394c992916f19837adb8364bec12245b61394a5b Mon Sep 17 00:00:00 2001 From: Teddy Do Date: Fri, 20 Nov 2026 15:53:27 -0800 Subject: [PATCH 15/15] Check the standalone EP round trip without a loss reduction Signed-off-by: Teddy Do --- tests/jax/test_te_ep_moe.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 2f6393c3e1..5f6c7d4a93 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -753,20 +753,27 @@ def test_standalone_dispatch_combine_default_layout(mesh): cfg = EpLayerConfig(top_k=TOPK, dispatch_output_per_expert_alignment=128) recv_capacity_per_rank = get_ep_config().recv_capacity_per_rank - def loss_fn(x, idx, w): + def round_trip(x, idx, w): recv_x, recv_w, handle_mem, token_counts, _ = ep_dispatch( cfg, idx, x, w, recv_capacity_per_rank ) recv_x, recv_w = (_shard_inputs(a, mesh) for a in (recv_x, recv_w)) expert_out = recv_x * recv_w[..., None].astype(recv_x.dtype) - out = ep_combine(cfg, handle_mem, token_counts, expert_out, num_tokens) - return 0.5 * jnp.sum(out.astype(jnp.float32) ** 2), out + return ep_combine(cfg, handle_mem, token_counts, expert_out, num_tokens) + + def fwd_bwd(x, idx, w): + # Pull the output back as its own cotangent (the gradient of half its squared + # norm) with no loss reduction, and pin both results to the token layout, so + # the EP ops are the only collectives. + out, vjp_fn = jax.vjp(lambda t: round_trip(t, idx, w), x) + (grad,) = vjp_fn(out) + return _shard_inputs(out, mesh), _shard_inputs(grad, mesh) resource = MeshResource(dp_resource=DP_RESOURCE, ep_resource=EP_AXIS, fsdp_resource=FSDP_AXIS) with mesh, global_shard_guard(resource): args = [_shard_inputs(a, mesh) for a in (tokens, topk_idx, topk_weights)] - (_, out), grad = jax.jit(jax.value_and_grad(loss_fn, has_aux=True))(*args) - # The loss is half the squared norm of the tokens, so its gradient is the tokens too. + out, grad = jax.jit(fwd_bwd)(*args) + # The round trip is the identity, so the output and its pulled-back cotangent are the tokens. expected = np.asarray(tokens, np.float32) for actual in (out, grad): np.testing.assert_allclose(