Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions tests/pytorch/test_sanity.py
Original file line number Diff line number Diff line change
Expand Up @@ -1651,3 +1651,53 @@ def check_weights():
with autocast(enabled=with_quantization, recipe=quantization_recipe):
y = module(x, **kwargs)
check_weights()


@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8)
def test_delayed_scaling_adapts_after_global_state_reset():
"""A module that outlives ``FP8GlobalStateManager.reset()`` keeps adapting.

The registration marker is a module-local key, so it used to survive a
``reset()`` and suppress registration into the replacement global buffers.
The module then recorded amax locally while the global buffer stayed empty,
and its delayed-scaling scale never updated again: with weights of 1/16 and
16 inputs of 128 the output stayed at 1.0 instead of 128.0. Reusing a module
after a reset is the sequential-runs-in-one-process case that ``reset()``
exists for.
"""
ds_recipe = recipe.DelayedScaling(amax_history_len=4, amax_compute_algo="most_recent")

def registered_entries():
qstate = FP8GlobalStateManager.quantization_state
return sum(len(entries) for entries in qstate.global_amax_buffer.values())

def step(amplitude):
model.zero_grad(set_to_none=True)
x = torch.full((16, 16), amplitude, dtype=torch.bfloat16, device="cuda", requires_grad=True)
with autocast(enabled=True, recipe=ds_recipe):
y = model(x)
y.sum().backward()
torch.cuda.synchronize()
return y.float().mean().item(), model.fp8_meta["scaling_fwd"].scale[0].item()

FP8GlobalStateManager.reset()
model = Linear(16, 16, bias=False, params_dtype=torch.bfloat16, device="cuda")
with torch.no_grad():
model.weight.fill_(1.0 / 16.0)

# Warm up so the module has registered forward and backward amax tensors.
step(1.0)
assert registered_entries() == 2, "warm-up should register forward and backward amax"

# Discard the state the module registered into and reuse the same module.
FP8GlobalStateManager.reset()
assert registered_entries() == 0
step(1.0)
assert registered_entries() == 2, "reused module did not register with the new global state"

# Delayed scaling costs one step of lag, then the output must follow the
# reference mean of 16 * 128 / 16 = 128.
out_mean_lagged, _ = step(128.0)
out_mean, _ = step(128.0)
assert out_mean_lagged < 2.0, "first step after the jump is expected to use the stale scale"
assert out_mean > 0.9 * 128.0, f"reused module did not adapt to the larger input: {out_mean}"
2 changes: 1 addition & 1 deletion transformer_engine/pytorch/module/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1523,7 +1523,7 @@ def to_cpu(src: torch.Tensor) -> torch.Tensor:
# Store other pickelable values
extra = {}
for k, v in self.fp8_meta.items():
if k != "buffer_index_and_autocast_key" and isinstance(
if k not in FP8GlobalStateManager.BOOKKEEPING_KEYS and isinstance(
v, (bool, int, float, str, tuple, list)
):
extra[k] = v
Expand Down
2 changes: 1 addition & 1 deletion transformer_engine/pytorch/ops/op.py
Original file line number Diff line number Diff line change
Expand Up @@ -856,7 +856,7 @@ def to_cpu(src: torch.Tensor) -> torch.Tensor:
# Store other picklable items
extra = {}
for key, val in fp8_meta.items():
if key == "buffer_index_and_autocast_key":
if key in FP8GlobalStateManager.BOOKKEEPING_KEYS:
continue
if not isinstance(val, (bool, int, float, str, tuple, list)):
continue
Expand Down
25 changes: 21 additions & 4 deletions transformer_engine/pytorch/quantization.py
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,10 @@ class FP8GlobalState:
is_first_fp8_module: bool = False
fp8_graph_capturing: bool = False
autocast_depth: int = 0
# Bumped on every reset(). A module records the generation it registered
# under, so a marker left over from a discarded state cannot suppress
# registration into the replacement state.
buffer_generation: int = 0
global_amax_buffer: Dict[str, list] = field(default_factory=dict)
global_amax_history_buffer: Dict[str, list] = field(default_factory=dict)
global_scale_buffer: Dict[str, list] = field(default_factory=dict)
Expand All @@ -482,6 +486,10 @@ class FP8GlobalStateManager:
FP8 state at different stages of execution.
"""

# `fp8_meta` carries these keys for bookkeeping, not as FP8 tensor state, so
# they are excluded from the pickled extra state of a module.
BOOKKEEPING_KEYS = ("buffer_index_and_autocast_key", "buffer_generation")

quantization_state = FP8GlobalState()

@classmethod
Expand All @@ -501,7 +509,11 @@ def get_skip_fp8_weight_update_tensor(cls) -> Optional[torch.Tensor]:
@classmethod
def reset(cls) -> None:
"""Reset the global state"""
cls.quantization_state = FP8GlobalState()
# The generation is carried into the replacement state so that modules
# which still hold its predecessor's registration marker register again.
cls.quantization_state = FP8GlobalState(
buffer_generation=cls.quantization_state.buffer_generation + 1
)

@classmethod
def is_fp8_available(cls) -> Tuple[bool, str]:
Expand Down Expand Up @@ -587,15 +599,20 @@ def add_fp8_tensors_to_global_buffer(
if not _has_delayed_scaling_state(fp8_meta):
return

# Every module must call this function exactly once since
# Every module must call this function exactly once per global state since
# the amax tensors are static. Ensures that compatibility
# with non-graphed modules is maintained.
index_in_buffer = cls.get_buffer_info() # Same index for fwd/bwd fp8 tensors.
if index_in_buffer in fp8_meta:
qstate = cls.quantization_state
if (
fp8_meta.get(index_in_buffer) is not None
and fp8_meta.get("buffer_generation") == qstate.buffer_generation
):
return

fp8_meta[index_in_buffer] = []
qstate = cls.quantization_state
fp8_meta["buffer_generation"] = qstate.buffer_generation

for forward in (True, False):
fp8_meta_tensor_key = cls.get_meta_tensor_key(forward=forward)
if fp8_meta_tensor_key not in fp8_meta:
Expand Down