diff --git a/tests/pytorch/test_sanity.py b/tests/pytorch/test_sanity.py index 2627de023a..39cdf47d18 100644 --- a/tests/pytorch/test_sanity.py +++ b/tests/pytorch/test_sanity.py @@ -1651,3 +1651,53 @@ def check_weights(): with autocast(enabled=with_quantization, recipe=quantization_recipe): y = module(x, **kwargs) check_weights() + + +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_delayed_scaling_adapts_after_global_state_reset(): + """A module that outlives ``FP8GlobalStateManager.reset()`` keeps adapting. + + The registration marker is a module-local key, so it used to survive a + ``reset()`` and suppress registration into the replacement global buffers. + The module then recorded amax locally while the global buffer stayed empty, + and its delayed-scaling scale never updated again: with weights of 1/16 and + 16 inputs of 128 the output stayed at 1.0 instead of 128.0. Reusing a module + after a reset is the sequential-runs-in-one-process case that ``reset()`` + exists for. + """ + ds_recipe = recipe.DelayedScaling(amax_history_len=4, amax_compute_algo="most_recent") + + def registered_entries(): + qstate = FP8GlobalStateManager.quantization_state + return sum(len(entries) for entries in qstate.global_amax_buffer.values()) + + def step(amplitude): + model.zero_grad(set_to_none=True) + x = torch.full((16, 16), amplitude, dtype=torch.bfloat16, device="cuda", requires_grad=True) + with autocast(enabled=True, recipe=ds_recipe): + y = model(x) + y.sum().backward() + torch.cuda.synchronize() + return y.float().mean().item(), model.fp8_meta["scaling_fwd"].scale[0].item() + + FP8GlobalStateManager.reset() + model = Linear(16, 16, bias=False, params_dtype=torch.bfloat16, device="cuda") + with torch.no_grad(): + model.weight.fill_(1.0 / 16.0) + + # Warm up so the module has registered forward and backward amax tensors. + step(1.0) + assert registered_entries() == 2, "warm-up should register forward and backward amax" + + # Discard the state the module registered into and reuse the same module. + FP8GlobalStateManager.reset() + assert registered_entries() == 0 + step(1.0) + assert registered_entries() == 2, "reused module did not register with the new global state" + + # Delayed scaling costs one step of lag, then the output must follow the + # reference mean of 16 * 128 / 16 = 128. + out_mean_lagged, _ = step(128.0) + out_mean, _ = step(128.0) + assert out_mean_lagged < 2.0, "first step after the jump is expected to use the stale scale" + assert out_mean > 0.9 * 128.0, f"reused module did not adapt to the larger input: {out_mean}" diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index ebc0d13895..9af4b0b657 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -1523,7 +1523,7 @@ def to_cpu(src: torch.Tensor) -> torch.Tensor: # Store other pickelable values extra = {} for k, v in self.fp8_meta.items(): - if k != "buffer_index_and_autocast_key" and isinstance( + if k not in FP8GlobalStateManager.BOOKKEEPING_KEYS and isinstance( v, (bool, int, float, str, tuple, list) ): extra[k] = v diff --git a/transformer_engine/pytorch/ops/op.py b/transformer_engine/pytorch/ops/op.py index 359c508888..7ca517e28c 100644 --- a/transformer_engine/pytorch/ops/op.py +++ b/transformer_engine/pytorch/ops/op.py @@ -856,7 +856,7 @@ def to_cpu(src: torch.Tensor) -> torch.Tensor: # Store other picklable items extra = {} for key, val in fp8_meta.items(): - if key == "buffer_index_and_autocast_key": + if key in FP8GlobalStateManager.BOOKKEEPING_KEYS: continue if not isinstance(val, (bool, int, float, str, tuple, list)): continue diff --git a/transformer_engine/pytorch/quantization.py b/transformer_engine/pytorch/quantization.py index 1c29914da8..50096898e4 100644 --- a/transformer_engine/pytorch/quantization.py +++ b/transformer_engine/pytorch/quantization.py @@ -467,6 +467,10 @@ class FP8GlobalState: is_first_fp8_module: bool = False fp8_graph_capturing: bool = False autocast_depth: int = 0 + # Bumped on every reset(). A module records the generation it registered + # under, so a marker left over from a discarded state cannot suppress + # registration into the replacement state. + buffer_generation: int = 0 global_amax_buffer: Dict[str, list] = field(default_factory=dict) global_amax_history_buffer: Dict[str, list] = field(default_factory=dict) global_scale_buffer: Dict[str, list] = field(default_factory=dict) @@ -482,6 +486,10 @@ class FP8GlobalStateManager: FP8 state at different stages of execution. """ + # `fp8_meta` carries these keys for bookkeeping, not as FP8 tensor state, so + # they are excluded from the pickled extra state of a module. + BOOKKEEPING_KEYS = ("buffer_index_and_autocast_key", "buffer_generation") + quantization_state = FP8GlobalState() @classmethod @@ -501,7 +509,11 @@ def get_skip_fp8_weight_update_tensor(cls) -> Optional[torch.Tensor]: @classmethod def reset(cls) -> None: """Reset the global state""" - cls.quantization_state = FP8GlobalState() + # The generation is carried into the replacement state so that modules + # which still hold its predecessor's registration marker register again. + cls.quantization_state = FP8GlobalState( + buffer_generation=cls.quantization_state.buffer_generation + 1 + ) @classmethod def is_fp8_available(cls) -> Tuple[bool, str]: @@ -587,15 +599,20 @@ def add_fp8_tensors_to_global_buffer( if not _has_delayed_scaling_state(fp8_meta): return - # Every module must call this function exactly once since + # Every module must call this function exactly once per global state since # the amax tensors are static. Ensures that compatibility # with non-graphed modules is maintained. index_in_buffer = cls.get_buffer_info() # Same index for fwd/bwd fp8 tensors. - if index_in_buffer in fp8_meta: + qstate = cls.quantization_state + if ( + fp8_meta.get(index_in_buffer) is not None + and fp8_meta.get("buffer_generation") == qstate.buffer_generation + ): return fp8_meta[index_in_buffer] = [] - qstate = cls.quantization_state + fp8_meta["buffer_generation"] = qstate.buffer_generation + for forward in (True, False): fp8_meta_tensor_key = cls.get_meta_tensor_key(forward=forward) if fp8_meta_tensor_key not in fp8_meta: