Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions tests/pytorch/test_permutation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1707,6 +1707,29 @@ def test_permutation_index_map(
)


def test_permute_with_probs_zero_probability_row_has_no_input_gradient():
"""A routed slot with probability zero is written as a zero row and must not backprop.

The forward zeroes that row (it is routing-map padding), so it does not depend on the input; a
gradient arriving for it must not be accumulated into the input tensor.
"""
device = "cuda"
routing_map = torch.tensor([[1, 1], [0, 1]], dtype=torch.int32, device=device)
probs = torch.tensor([[1.0, 0.0], [0.0, 1.0]], device=device)
x = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, requires_grad=True)

out, _, _ = te_permute_with_probs(x, probs, routing_map, 3)
row_weights = torch.tensor([[1.0, 1.0], [10.0, 10.0], [100.0, 100.0]], device=device)
(out * row_weights).sum().backward()

x_ref = x.detach().clone().requires_grad_(True)
reference = torch.stack([x_ref[0], torch.zeros_like(x_ref[0]), x_ref[1]])
(reference * row_weights).sum().backward()

torch.testing.assert_close(out, reference)
torch.testing.assert_close(x.grad, x_ref.grad)


@pytest.mark.parametrize("te_dtype", _te_dtypes)
@pytest.mark.parametrize("use_torch_compile", [False, True])
def test_permutation_index_map_empty_input(te_dtype, use_torch_compile):
Expand Down
15 changes: 13 additions & 2 deletions transformer_engine/pytorch/permutation.py
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ def _moe_permute_mask_map_setup_context(ctx, inputs, output):
inp, routing_map, _num_out_tokens, probs, pad_offsets = inputs
_output_tensor, row_id_map, _permuted_probs = output
ctx.empty_input = inp.size(0) == 0
ctx.save_for_backward(row_id_map, pad_offsets)
ctx.save_for_backward(row_id_map, pad_offsets, _permuted_probs)
ctx.num_experts = routing_map.size(1)
ctx.num_tokens = inp.size(0)
ctx.hidden_size = inp.size(1) if not ctx.empty_input else 0
Expand All @@ -558,7 +558,18 @@ def _moe_permute_mask_map_backward_wrapper(
grad_output, QuantizedTensor
), "The backward of moe_permute does not support FP8."

row_id_map, pad_offsets = ctx.saved_tensors
row_id_map, pad_offsets, permuted_probs = ctx.saved_tensors

# The forward writes a zero row for a routed slot whose probability is zero (see the
# PERMUTE_PROBS branch of the kernel), so that row does not depend on the input and its
# incoming gradient must not reach it. Without this the padded row contributes to the
# input gradient even though it contributed nothing to the output.
if permuted_probs.numel() > 0:
# Select rather than multiply: an infinite gradient in a masked row would survive as
# inf * 0 = NaN, which would then leak into the accumulation.
grad_output = torch.where(
(permuted_probs != 0).unsqueeze(-1), grad_output, torch.zeros_like(grad_output)
)

# Pass permuted_probs_grad only if it has content
probs_grad_input = grad_permuted_probs if grad_permuted_probs.numel() > 0 else None
Expand Down
Loading