Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 182 additions & 16 deletions tests/pytorch/debug/test_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
from transformer_engine.debug.features.utils.stats_computation import (
compute_max_blockwise_dynamic_range,
BlockwiseDynamicRangeStat,
STATS,
stats_to_num,
)
import math

Expand Down Expand Up @@ -61,6 +63,7 @@
"underflows%",
"scale_inv_min",
"scale_inv_max",
"scale_inv_std",
"mse",
]

Expand Down Expand Up @@ -248,6 +251,11 @@ def test_log_quantized_stats_numerics(fp8_recipe, feature_dirs):
debug_api.step()

dequantized_tensor = quantized_tensor.dequantize()
if hasattr(quantized_tensor, "_scale_inv"):
scale_inv_rowwise = quantized_tensor._scale_inv.float()
else:
scale_inv_rowwise = quantized_tensor._rowwise_scale_inv.float()
scale_inv_rowwise = scale_inv_rowwise[scale_inv_rowwise != 0]
output = read_log(log_dir)

for line in output.splitlines():
Expand All @@ -267,6 +275,17 @@ def test_log_quantized_stats_numerics(fp8_recipe, feature_dirs):
(abs(dequantized_tensor) > abs(tensor)).sum() / dequantized_tensor.numel() * 100
)
assert overflows == pytest.approx(expected.cpu(), abs=1e-4)
# Rowwise scale_inv stats only; logger formats with {:.4f} so abs<1e-4.
if "scale_inv_min" in line and "_columnwise" not in line:
value = float(line.split("value=")[1])
assert value == pytest.approx(scale_inv_rowwise.min().cpu().item(), abs=1e-4)
if "scale_inv_max" in line and "_columnwise" not in line:
value = float(line.split("value=")[1])
assert value == pytest.approx(scale_inv_rowwise.max().cpu().item(), abs=1e-4)
if "scale_inv_std" in line and "_columnwise" not in line:
value = float(line.split("value=")[1])
expected = torch.std(scale_inv_rowwise, unbiased=False).cpu().item()
assert value == pytest.approx(expected, abs=1e-4)


LOG_HIGH_PRECISION_CONFIG = """
Expand Down Expand Up @@ -370,6 +389,121 @@ def test_log_stats_numerics(feature_dirs, tensor_name):
assert found_dynamic_range, "dynamic_range not found in output"


def test_stats_computation_microbatch_reduction():
"""Reducing per-microbatch stats must equal computing them over the concatenation.

Sweeps every aux-free (high precision) stat in the registry, using its own
compute/combine fns, with microbatches of different means and unequal sizes to
stress the parallel variance combine that uniform ~N(0,1) tests miss.
"""
torch.manual_seed(0)
microbatches = [
0.5 * torch.randn(4, 4).cuda() + 0.0,
0.5 * torch.randn(8, 4).cuda() + 8.0,
0.5 * torch.randn(6, 4).cuda() - 6.0,
]
full = torch.cat([mb.flatten() for mb in microbatches])

# Aux-free stats compute from the tensor alone; the rest need aux_dict
# (quantized tensors) and raise on {}, so they are out of scope here.
def is_aux_free(stat):
if isinstance(stat, BlockwiseDynamicRangeStat):
return False # parametrized stat, covered by its own direct test
try:
STATS[stat][0](full, {})
return True
except Exception: # pylint: disable=broad-except
return False

aux_free = [stat for stat in stats_to_num if is_aux_free(stat)]

# Fill a [num_microbatches, num_stats] buffer exactly like _Buffer would, then
# check every combinator against its own compute fn over the concatenation.
buffers = torch.zeros(len(microbatches), len(stats_to_num)).cuda()
for i, mb in enumerate(microbatches):
for stat in aux_free:
buffers[i, stats_to_num[stat]] = float(STATS[stat][0](mb, {}))

for stat in aux_free:
reduced = float(STATS[stat][1](buffers))
expected = float(STATS[stat][0](full, {}))
assert reduced == pytest.approx(
expected, rel=1e-4, abs=1e-4
), f"{stat}: reduced {reduced}, expected {expected}"


@pytest.mark.parametrize(
"fp8_recipe, recipe_name",
[
pytest.param(recipe.DelayedScaling(), "fp8_delayed_scaling", id="delayed"),
pytest.param(recipe.Float8CurrentScaling(), "fp8_current_scaling", id="current"),
pytest.param(recipe.MXFP8BlockScaling(), "mxfp8", id="mxfp8"),
pytest.param(recipe.Float8BlockScaling(), "fp8_block_scaling", id="fp8_block_scaling"),
pytest.param(recipe.NVFP4BlockScaling(), "nvfp4", id="nvfp4"),
],
)
@pytest.mark.parametrize("columnwise", [False, True])
@pytest.mark.parametrize("padded", [False, True])
def test_scale_inv_std_microbatch_reduction(fp8_recipe, recipe_name, columnwise, padded):
"""Scale statistics exclude padding and combine across unequal microbatches."""
if not fp8_available:
pytest.skip(reason_for_no_fp8)
if recipe_name == "mxfp8" and not mxfp8_available:
pytest.skip(reason_for_no_mxfp8)
if recipe_name == "fp8_block_scaling" and not fp8_block_scaling_available:
pytest.skip(reason_for_no_fp8_block_scaling)
if recipe_name == "nvfp4" and not nvfp4_available:
pytest.skip(reason_for_no_nvfp4)
if columnwise and recipe_name in ["fp8_delayed_scaling", "fp8_current_scaling"]:
pytest.skip("Per-tensor recipes have no columnwise scale statistics")
if padded and recipe_name not in ["mxfp8", "nvfp4"]:
pytest.skip("Only MXFP8 and NVFP4 scale buffers need padding")

torch.manual_seed(0)
shapes = [(160, 288), (288, 160), (96, 416)] if padded else [(256, 1024), (512, 1024)]
suffix = "_columnwise" if columnwise else ""
names = {
stat: f"{recipe_name}_scale_inv_{stat}{suffix}"
for stat in ("min", "max", "std", "variance", "numel", "sum")
}
buffers = torch.zeros(len(shapes), len(stats_to_num)).cuda()
scale_invs = []
for i, shape in enumerate(shapes):
tensor = (i + 1) * torch.randn(shape, device="cuda", dtype=torch.bfloat16) + 8 * i
recipe_state = RecipeState.create(fp8_recipe, mode="forward", num_quantizers=1)
quantizer = recipe_state.make_quantizers()[0]
quantized_tensor = quantizer(tensor)
if hasattr(quantized_tensor, "_scale_inv"):
scales = quantized_tensor._scale_inv
else:
scales = (
quantized_tensor._columnwise_scale_inv
if columnwise
else quantized_tensor._rowwise_scale_inv
)
scales = scales.float().flatten()
if padded:
assert torch.any(scales == 0), "Expected zero-padded scale storage"
scales = scales[scales != 0]
scale_invs.append(scales)
aux_dict = {recipe_name: quantized_tensor}
for stat in names.values():
buffers[i, stats_to_num[stat]] = STATS[stat][0](tensor, aux_dict)
assert float(buffers[i, stats_to_num[names["std"]]]) == pytest.approx(
float(torch.std(scales, unbiased=False)), rel=1e-4, abs=1e-4
)

scales = torch.cat(scale_invs)
for stat, expected in (
("min", scales.min()),
("max", scales.max()),
("std", torch.std(scales, unbiased=False)),
("numel", scales.numel()),
):
reduced = float(STATS[names[stat]][1](buffers))
assert reduced == pytest.approx(float(expected), rel=1e-4, abs=1e-4), names[stat]


@pytest.mark.parametrize("layer", ["linear", "transformer"])
def test_log_every_3_or_5_layers(layer, configs_dir, feature_dirs):
if not fp8_available:
Expand Down Expand Up @@ -403,7 +537,8 @@ def test_log_every_3_or_5_layers(layer, configs_dir, feature_dirs):

with open(
os.path.join(
temp_dir, "nvdlfw_inspect_statistics_logs/nvdlfw_inspect_globalrank-0.log"
temp_dir,
"nvdlfw_inspect_statistics_logs/nvdlfw_inspect_globalrank-0.log",
),
"r",
) as f:
Expand Down Expand Up @@ -443,7 +578,14 @@ def test_nvfp4_numeric(feature_dirs):
if not nvfp4_available:
pytest.skip(reason_for_no_nvfp4)

log_nvfp4_config = LOG_NVFP4_CONFIG_BASE.format(stats="underflows%, mse")
scale_stats = [
f"scale_inv_{stat}{suffix}"
for stat in ("min", "max", "std")
for suffix in ("", "_columnwise")
]
log_nvfp4_config = LOG_NVFP4_CONFIG_BASE.format(
stats=", ".join(["underflows%", "mse"] + scale_stats)
)

with debug_session(log_nvfp4_config, feature_dirs) as log_dir:
recipe_state = RecipeState.create(
Expand Down Expand Up @@ -476,7 +618,24 @@ def test_nvfp4_numeric(feature_dirs):
dequantized_tensor = quantized_tensor.dequantize()
output = read_log(log_dir)

# Validate both stats are present
values = {
line.split("nvfp4_", 1)[1].split()[0]: float(line.split("value=")[1].split()[0])
for line in output.splitlines()
if "nvfp4_scale_inv_" in line and "value=" in line
}
for suffix, scale_inv in (
("", quantized_tensor._rowwise_scale_inv),
("_columnwise", quantized_tensor._columnwise_scale_inv),
):
scales = scale_inv.float()
scales = scales[scales != 0]
for stat, expected in (
("min", scales.min()),
("max", scales.max()),
("std", torch.std(scales, unbiased=False)),
):
assert values[f"scale_inv_{stat}{suffix}"] == pytest.approx(float(expected), abs=1e-4)

assert "nvfp4_underflows%" in output, "underflows% stat missing"
assert "nvfp4_mse" in output, "mse stat missing"

Expand Down Expand Up @@ -508,29 +667,36 @@ def test_nvfp4_numeric(feature_dirs):
assert mse_value == pytest.approx(expected_mse.cpu().item(), abs=1e-4)


def test_fp8_stats_allows_nvfp4_with_recipe_prefix(feature_dirs):
"""Test that LogFp8TensorStats allows recipe-prefixed stats with NVFP4 for what-if analysis."""
@pytest.mark.parametrize("fp8_stats", ["mxfp8_mse", "scale_inv_std, mxfp8_mse", "scale_inv_std"])
def test_fp8_stats_allows_nvfp4_with_recipe_prefix(feature_dirs, fp8_stats):
"""FP8 what-if and NVFP4 logging can share a layer selection."""
if not nvfp4_available:
pytest.skip(reason_for_no_nvfp4)

# Use recipe-prefixed stat with NVFP4 - should work (computes MXFP8 separately)
log_fp8_config = LOG_QUANTIZED_CONFIG_BASE.format(stats="mxfp8_mse")
log_fp8_config = LOG_QUANTIZED_CONFIG_BASE.format(stats=fp8_stats)
log_fp8_config += LOG_NVFP4_CONFIG_BASE.format(stats="scale_inv_std").split(
" transformer_engine:\n", 1
)[1]

with debug_session(log_fp8_config, feature_dirs) as log_dir:
model = te.Linear(128, 128, params_dtype=torch.bfloat16)
inp = torch.randn(128, 128, dtype=torch.bfloat16).cuda()

# Should work - recipe-prefixed stats compute MXFP8 separately for comparison
for _ in range(2):
with te.autocast(recipe=recipe.NVFP4BlockScaling()):
output = model(inp)
loss = output.sum()
loss.backward()
debug_api.step()
warning_context = (
pytest.warns(UserWarning, match="Skipping stats .*layer uses NVFP4")
if "scale_inv_std" in fp8_stats
else contextlib.nullcontext()
)
with warning_context:
for _ in range(2):
with te.autocast(recipe=recipe.NVFP4BlockScaling()):
output = model(inp)
output.sum().backward()
debug_api.step()

output = read_log(log_dir)
# Should have logged MXFP8 MSE stat (what-if scenario)
assert "mxfp8_mse" in output
assert "nvfp4_scale_inv_std" in output
assert ("mxfp8_mse" in output) == ("mxfp8_mse" in fp8_stats)


def test_log_grouped_gemm(feature_dirs):
Expand Down
41 changes: 29 additions & 12 deletions transformer_engine/debug/features/log_fp8_tensor_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,13 @@ class LogFp8TensorStats(BaseLogTensorStats):
- overflows% - percentage of elements of tensor that were clipped to the max/min value of the FP8 range - supported only for fp8_delayed_scaling,
- scale_inv_min - minimum of the inverse of the scaling factors,
- scale_inv_max - maximum of the inverse of the scaling factors,
- scale_inv_std - population standard deviation of the inverse of the scaling factors;
useful for spotting clipping that min/max alone can miss (degenerate to 0 for
fp8_delayed_scaling / fp8_current_scaling since those use a single scalar scale).
- mse - mean squared error of the quantized tensor and the original tensor = sum((quantized_tensor - original_tensor)**2) / num_elements,

When collecting stats for the weight tensor with FP8 model parameters enabled,
only "scale_inv_min" and "scale_inv_max" are available.
only "scale_inv_min", "scale_inv_max" and "scale_inv_std" are available.
All other statistics require access to the high precision tensor.

tensors/tensors_struct: List[str]
Expand Down Expand Up @@ -188,19 +191,12 @@ def check_if_stat_is_supported(
f"Stat {stat} does not contain a recipe name and the current recipe is not set."
)

if recipe_from_stat == "nvfp4":
raise ValueError(f"Stat {stat} requires LogNvfp4TensorStats for NVFP4 tensors.")

if recipe_from_stat != "" and recipe_from_stat not in ALL_RECIPE_NAMES:
raise ValueError(f"Stat {stat} contains an unsupported recipe name: {recipe_from_stat}")

# Block any NVFP4 stats in LogFp8TensorStats (FP8-specific logic won't work)
# But allow recipe-prefixed FP8 stats like "mxfp8_underflows%" even with NVFP4 quantizer
if recipe_from_stat == "nvfp4":
raise ValueError(
f"[NVTORCH INSPECT ERROR] Cannot compute NVFP4 stats '{stat}' in LogFp8TensorStats."
" FP8-specific statistics do not work with NVFP4. Use LogNvfp4TensorStats for"
" NVFP4-specific stats, or use FP8 recipe-prefixed stats (e.g.,"
" 'mxfp8_underflows%', 'fp8_block_scaling_mse') for what-if FP8 comparisons."
)

if recipe_from_stat in ["fp8_delayed_scaling", "fp8_current_scaling"] and columnwise:
raise ValueError(
f"Stat {stat} is not supported. Columnwise tensor statistics are not supported for"
Expand All @@ -216,7 +212,7 @@ def check_if_stat_is_supported(
if recipe_from_stat == "mxfp8" and torch.cuda.get_device_capability()[0] < 10:
raise ValueError(f"Stat {stat} needs Blackwell or later GPU.")

supported_stats = ["underflows%", "scale_inv_min", "scale_inv_max", "mse"]
supported_stats = ["underflows%", "scale_inv_min", "scale_inv_max", "scale_inv_std", "mse"]
if stat_without_recipe not in supported_stats:
raise ValueError(
f"Stat {stat} contains an unsupported stat name: {stat_without_recipe}"
Expand Down Expand Up @@ -338,6 +334,27 @@ def inspect_tensor(

recipe_name = _get_recipe_name(quantizer)

# If the layer uses NVFP4, drop bare stats (which would target the NVFP4
# recipe that LogFp8TensorStats can't handle) but keep stats explicitly
# prefixed with an FP8 recipe (e.g. "mxfp8_mse") for what-if FP8 comparison.
if _nvfp4_available and isinstance(quantizer, NVFP4Quantizer):
kept_stats, dropped_stats = [], []
for stat in config["stats"]:
if any(r in stat for r in ALL_RECIPE_NAMES):
kept_stats.append(stat)
else:
dropped_stats.append(stat)
if dropped_stats:
warnings.warn(
f"[LogFp8TensorStats] Skipping stats {dropped_stats} for layer "
f"'{layer_name}', tensor '{tensor_name}': layer uses NVFP4. Use "
"LogNvfp4TensorStats for NVFP4 stats, or prefix stats with an FP8 "
"recipe name (e.g. 'mxfp8_mse') for what-if FP8 comparisons."
)
if not kept_stats:
return
config = {**config, "stats": kept_stats}

for stat in config["stats"]:
self.check_if_stat_is_supported(
stat, recipe_name, high_precision_tensor_provided=tensor is not None
Expand Down
24 changes: 19 additions & 5 deletions transformer_engine/debug/features/log_nvfp4_tensor_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ class LogNvfp4TensorStats(BaseLogTensorStats):
List of statistics to collect. Available stats:
- underflows% - percentage of non-zero elements clipped to 0 (from packed FP4 data)
- mse - mean squared error = sum((quantized_tensor - original_tensor)**2) / num_elements
- scale_inv_min - minimum of the inverse of the scaling factors
- scale_inv_max - maximum of the inverse of the scaling factors
- scale_inv_std - population standard deviation of the inverse of the scaling factors;
useful for spotting clipping that min/max alone can miss

tensors/tensors_struct: List[str]
list of tensors to log
Expand Down Expand Up @@ -85,13 +89,23 @@ class LogNvfp4TensorStats(BaseLogTensorStats):

def check_if_stat_is_supported(self, stat: str):
"""Returns True if stat is supported, raises ValueError otherwise."""
supported_stats = [
"underflows%",
"mse",
]
if stat not in supported_stats:
# Only scale_inv_* stats have a columnwise variant (separate rowwise/columnwise
# scale_inv); underflows%/mse are computed from the single quantized tensor and
# have no '_columnwise' form, so they must not accept the suffix.
columnwise_stats = ["scale_inv_min", "scale_inv_max", "scale_inv_std"]
supported_stats = ["underflows%", "mse"] + columnwise_stats

if stat.endswith("_columnwise"):
bare = stat[: -len("_columnwise")]
if bare not in columnwise_stats:
raise ValueError(
f"Stat {stat} is not supported for NVFP4. The '_columnwise' suffix is only"
f" valid for {columnwise_stats}."
)
elif stat not in supported_stats:
raise ValueError(
f"Stat {stat} is not supported for NVFP4. Supported stats: {supported_stats}"
" (scale_inv_* may take an optional '_columnwise' suffix)."
)
return True

Expand Down
Loading
Loading