Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions qa/L0_jax_distributed_unittest/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ wait
# >=4 visible GPUs.
TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \
|| test_fail "test_te_ep_moe.py"
# Same suite with single-axis EP: dp=2 x fsdp=2 x expert=2 with 8 or more GPUs,
# else fsdp x expert, so that fsdp stays above 1 on either runner.
EP_MOE_DP_SIZE=1
if [ "${NUM_GPUS:-$(nvidia-smi -L | wc -l)}" -ge 8 ]; then
EP_MOE_DP_SIZE=2
fi
TE_EP_MOE_DP_SIZE=$EP_MOE_DP_SIZE TE_EP_MOE_TP_SIZE=1 TE_PATH=$TE_PATH bash $TE_PATH/tests/jax/run_te_ep_moe.sh \
|| test_fail "test_te_ep_moe.py (TP=1)"
# Exercise the multi-GPU tutorial in docs/examples/jax (needs >= 4 GPUs;
# auto-skips otherwise).
CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -m pytest -c $TE_PATH/tests/jax/pytest.ini -v --junitxml=$XML_LOG_DIR/pytest_docs_examples_jax_distributed.xml -k multi_gpu $TE_PATH/docs/examples/jax/ || test_fail "docs/examples/jax (multi-GPU)"
Expand Down
91 changes: 81 additions & 10 deletions tests/jax/moe/test_moe_resource_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,50 @@ def test_explicit_resource_overrides_global_and_is_snapshotted():
assert quantize is True


def test_tuple_ep_resource():
resource, _ = _resolve_moe_mesh_resource(
MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource=("ep", "tp"))
)
assert _moe_mesh_axes(resource) == (("ep", "tp"), ("dp", "fsdp"))
with pytest.raises(ValueError, match="distinct"):
_resolve_moe_mesh_resource(MeshResource(fsdp_resource="tp", ep_resource=("ep", "tp")))


@pytest.mark.parametrize(
"resource,outer_axes",
[
(
MeshResource(
dp_resource="dp", fsdp_resource=("ep", "fsdp", "tp"), ep_resource=("ep", "tp")
),
("dp", "fsdp"),
),
(MeshResource(dp_resource="tp", fsdp_resource="ep", ep_resource=("ep", "tp")), ()),
],
)
def test_ep_outer_axes_flatten_tuple_resources(resource, outer_axes):
"""EP outer axes flatten a tuple FSDP resource and drop every axis of a tuple EP."""
from transformer_engine.jax.cpp_extensions import ep

with global_shard_guard(resource):
assert ep._capture_ep_resource_axes() == (("ep", "tp"), outer_axes)


def test_tuple_resources_flatten_in_default_spec_and_sizes(monkeypatch):
"""Tuple FSDP and EP resources give a flat default EP output spec and product sizes."""
from types import SimpleNamespace
from transformer_engine.jax import sharding
from transformer_engine.jax.ep import _default_out_partition_spec

mesh = SimpleNamespace(shape={"fsdp": 2, "data": 2, "ep": 2, "tp": 2}, empty=False)
monkeypatch.setattr(sharding, "_get_mesh", lambda: mesh)
resource = MeshResource(fsdp_resource=("fsdp", "data", "ep"), ep_resource=("ep", "tp"))
with global_shard_guard(resource):
assert _default_out_partition_spec() == (("fsdp", "data", "ep", "tp"),)
assert sharding.ep_axis_size() == 4
assert sharding.get_active_resource_axis("ep_resource") == ("ep", "tp")


def test_global_default_and_duplicate_outer_axis():
with global_shard_guard(
MeshResource(dp_resource="fsdp", fsdp_resource="fsdp", ep_resource="ep")
Expand Down Expand Up @@ -89,6 +133,23 @@ def test_ep_leading_axis_canonicalization(ep_size, expected):
assert not ep._leading_axis_ok(P("fsdp", None, None), axes, missing_ep_mesh)[0]


@pytest.mark.parametrize("tp_size,expected", [(1, True), (2, False)])
def test_tuple_ep_leading_axis_canonicalization(tp_size, expected):
"""A tuple EP axis may be absent from the spec only when its mesh size is one."""
from types import SimpleNamespace
from jax.sharding import PartitionSpec as P
from transformer_engine.jax.cpp_extensions import ep

mesh = SimpleNamespace(shape={"fsdp": 2, "ep": 2, "tp": tp_size})
axes = (("ep", "tp"), ("fsdp",))
assert ep._leading_axis_ok(P(("fsdp", "ep"), None, None), axes, mesh)[0] is expected
assert ep._leading_axis_ok(P("ep", None, None), axes, mesh)[0] is expected
assert ep._leading_axis_ok(P(("fsdp", "ep", "tp"), None, None), axes, mesh)[0]
assert not ep._leading_axis_ok(P(("fsdp", "tp"), None, None), axes, mesh)[0]
assert not ep._leading_axis_ok(P(None, None, None), axes, mesh)[0]
assert not ep._leading_axis_ok(P(("fsdp", "ep"), None, None), axes)[0]


def test_fused_ep_partitioning_retains_bind_time_resources(monkeypatch):
from types import SimpleNamespace
from jax.sharding import NamedSharding, PartitionSpec as P
Expand Down Expand Up @@ -126,35 +187,41 @@ def bind(*args, **kwargs):
"fsdp_resource,outer_axis",
[("fsdp", "fsdp"), (("fsdp", "ep"), "fsdp"), (("ep", "fsdp"), "fsdp"), (("ep",), None)],
)
def test_ep_outer_axis_excludes_ep_from_fsdp_tuple(monkeypatch, fsdp_resource, outer_axis):
from types import SimpleNamespace
from transformer_engine.jax import sharding
from transformer_engine.jax.cpp_extensions import ep
def test_default_ep_out_spec_excludes_ep_from_fsdp_tuple(fsdp_resource, outer_axis):
from transformer_engine.jax.ep import _default_out_partition_spec

monkeypatch.setattr(sharding, "_get_mesh", lambda: SimpleNamespace(shape={"fsdp": 2, "ep": 4}))
with global_shard_guard(MeshResource(fsdp_resource=fsdp_resource, ep_resource="ep")):
assert ep._ep_outer_axis() == outer_axis
leading = (outer_axis, "ep") if outer_axis is not None else "ep"
assert _default_out_partition_spec() == (leading,)


def test_default_ep_out_spec_flattens_two_outer_fsdp_axes(monkeypatch):
def test_default_ep_out_spec_flattens_two_outer_fsdp_axes():
from types import SimpleNamespace
from transformer_engine.jax import sharding
from transformer_engine.jax.cpp_extensions import ep
from transformer_engine.jax.ep import _default_out_partition_spec

mesh = SimpleNamespace(shape={"fsdp1": 2, "fsdp2": 2, "ep": 2})
monkeypatch.setattr(sharding, "_get_mesh", lambda: mesh)
resource = MeshResource(fsdp_resource=("fsdp1", "fsdp2", "ep"), ep_resource="ep")
with global_shard_guard(resource):
assert ep._ep_outer_axis() == ("fsdp1", "fsdp2")
spec = _default_out_partition_spec()
assert spec == (("fsdp1", "fsdp2", "ep"),)
assert ep._leading_per_shard((16, 4), spec[0], mesh) == (2, 4)


def test_default_ep_out_spec_uses_dp_and_fsdp():
"""The default EP out spec shards over dp and fsdp, as ep_bootstrap counts EP groups."""
from types import SimpleNamespace
from transformer_engine.jax.cpp_extensions import ep
from transformer_engine.jax.ep import _default_out_partition_spec

mesh = SimpleNamespace(shape={"dp": 2, "fsdp": 2, "ep": 2})
resource = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep")
with global_shard_guard(resource):
spec = _default_out_partition_spec()
assert spec == (("dp", "fsdp", "ep"),)
assert ep._leading_per_shard((16, 4), spec[0], mesh) == (2, 4)


@pytest.mark.parametrize(
"resource,quantize,error",
[
Expand All @@ -169,10 +236,14 @@ def test_invalid_resource(resource, quantize, error):


def test_bool_and_resource_types():
from transformer_engine.jax.ep import ep_bootstrap

with pytest.raises(TypeError, match="must be a bool"):
_resolve_moe_mesh_resource(MeshResource(ep_resource="ep"), "true")
with pytest.raises(TypeError, match="must be a MeshResource"):
_resolve_moe_mesh_resource("ep")
with pytest.raises(TypeError, match="must be a MeshResource"):
ep_bootstrap(None, None, 2, 1, 1, 1, mesh_resource="ep")


@pytest.mark.parametrize("quantize", [False, True])
Expand Down
3 changes: 2 additions & 1 deletion tests/jax/run_te_ep_moe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# its own --num-process=N --process-id=i, and waits for all of them. Each
# child calls jax.distributed.initialize(..., local_device_ids=process_id)
# so each Python process only sees its one GPU as a local device and the
# participating processes form a global (ep, fsdp) mesh.
# participating processes form a global ([dp,] fsdp, expert[, tensor]) mesh.

set -euo pipefail

Expand Down Expand Up @@ -38,6 +38,7 @@ echo "============================================================"
echo "TE-EP MoE MULTIPROCESS test (one process per GPU, ${NUM_GPUS} GPUs)"
echo " test file : $TEST_FILE"
echo " coordinator : $TE_EP_MOE_COORDINATOR_ADDRESS"
echo " DP / TP size / input rank: ${TE_EP_MOE_DP_SIZE:-1} / ${TE_EP_MOE_TP_SIZE:-2} / ${TE_EP_MOE_INPUT_RANK:-auto}"
echo " XLA_PYTHON_CLIENT_PREALLOCATE: $XLA_PYTHON_CLIENT_PREALLOCATE"
echo " XLA_PYTHON_CLIENT_MEM_FRACTION: $XLA_PYTHON_CLIENT_MEM_FRACTION"
if [ "$#" -gt 0 ]; then
Expand Down
29 changes: 28 additions & 1 deletion tests/jax/test_multi_process_ep.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@
ep_dispatch_fwd,
ep_combine_fwd,
get_ep_config,
_leading_axis_ok,
_ep_spec_ok,
is_ep_borrowed_comm_built,
use_nccl_comm_from_xla,
)
Expand All @@ -62,7 +64,6 @@
is_xla_ffi_collectives_supported,
)


# ── Test config ─────────────────────────────────────────────────────────────
# NCCL EP requires NUM_LOCAL_EXPERTS*ep % 4 == 0 (TMA alignment in
# device/hybridep_adapter.cu:511). With NUM_LOCAL_EXPERTS=2, ep must be even.
Expand Down Expand Up @@ -1418,6 +1419,32 @@ def test_ep_tp_splits_domains(self):

self.assertEqual(domains, {0: [0, 2, 4, 6], 1: [1, 3, 5, 7]})

def test_compound_ep_axis(self):
if not is_devices_enough(4):
self.skipTest("requires 4 devices")
mesh = Mesh(np.asarray(jax.devices()[:4]).reshape(2, 2), ("expert", "tensor"))
order = {int(device.id): i for i, device in enumerate(mesh.devices.reshape(-1))}

domains = {}
for rank in range(4):
root, col, num_domains = _ep_domain_for_rank(
mesh,
("expert", "tensor"),
rank,
device_to_rank=lambda device: order[int(device.id)],
)
self.assertEqual(num_domains, 1)
domains.setdefault(root, {})[col] = rank
self.assertEqual([domains[0][i] for i in range(4)], [0, 1, 2, 3])

ep_axes = ("expert", "tensor")
with mesh, global_shard_guard(MeshResource(ep_resource=ep_axes)):
ok, ep_axis, outer_axes = _leading_axis_ok(PartitionSpec(ep_axes, None, None))
self.assertTrue(ok)
self.assertEqual(ep_axis, ("expert", "tensor"))
self.assertEqual(outer_axes, ())
self.assertTrue(_ep_spec_ok(PartitionSpec(ep_axes, None, None), 2))

def test_ep_flattened_replica_groups_multi_domain(self):
"""Verify flattened replica IDs are grouped by DP coordinate."""
if not is_devices_enough(4):
Expand Down
Loading
Loading