diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py index f631e208fc40..494db913ae6c 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/backend.py @@ -21,13 +21,20 @@ ) from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm._torch.modules.multi_stream_utils import do_multi_stream, private_stream -from tensorrt_llm._utils import get_sm_version, is_sm_100f +from tensorrt_llm._utils import get_sm_version, is_blackwell_datacenter, is_sm_100f from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo from .indexer import CSA2Indexer from .metadata import CSA2TrtllmMetadata -from .params import CSA2BackendForwardArgs, CSA2Layout, CSA2Mode, CSA2Params, select_csa2_backend +from .params import ( + CSA2BackendForwardArgs, + CSA2ForwardState, + CSA2Layout, + CSA2Mode, + CSA2Params, + select_csa2_backend, +) if TYPE_CHECKING: from flashinfer.prefill import ( @@ -408,6 +415,14 @@ def direct_decode_serves(self, rows, context_tokens, main_width, device) -> bool and direct_decode_device_supported(device) ) + def direct_decode_carveout(self, main_width, device) -> int: + """Shared-memory carveout (percent) the direct decode of this geometry runs at; 0 for the + driver's choice.""" + from .direct_decode import _ROOM_CARVEOUT, _v2_engaged + + engaged = _v2_engaged(self.num_heads, _SWA_TILE, main_width, device.index) + return _ROOM_CARVEOUT if engaged else 0 + def gen_fmha_applies(self, q, metadata, forward_args) -> bool: """Whether this staged tile runs the ``gen_fmha`` generation core instead of the native cubin: a ``gen_fmha_layer``'s generation tile of a forward without context tokens @@ -469,7 +484,7 @@ def _forward_gen_fmha(self, q, metadata, forward_args): def _forward_direct(self, q, metadata, forward_args): """Publish this layer's rows (and run its indexer), then attend the selected packed rows in one launch. Nothing is staged, so no staging scale is derived.""" - from .direct_decode import direct_sparse_decode + from .direct_decode import direct_sparse_decode, main_rows_bytes self._validate_helper_output(q, forward_args) module = forward_args.sparse_backend_args @@ -484,9 +499,31 @@ def _forward_direct(self, q, metadata, forward_args): query = q.view(metadata.num_tokens, self.num_heads, _HEAD_DIM) output = forward_args.output state = module.state - # Exported slots are final like the selections they map, so readiness keeps the mapping's. - main_ready = ready and reuse and inputs.main_mapping is not None - inputs, slots_out = self._exported_slots(inputs, metadata, module) + # A Reuse layer's main rows, selections and exported slots come from earlier layers: final + # once the cache write launched here has passed its own dependency wait, query ready or not. + main_ready = ( + (ready or (rows_written and is_blackwell_datacenter())) + and reuse + and inputs.main_mapping is not None + ) + # SM100/SM103 attention TP: a Full layer's decode starts once its selections are final; a + # programmatic edge from the indexer's stream would start it early but resume it late. + mapping = getattr(metadata, "mapping", None) + plain_edge = ( + layout is not None + and layout.layer(self.layer_idx).mode == CSA2Mode.FULL + and is_blackwell_datacenter() + and not (mapping is not None and mapping.enable_attention_dp) + ) + image = 0 + if inputs.main_mapping is not None and inputs.topk_indices is not None: + image = main_rows_bytes( + self.num_heads, + int(inputs.swa_indices.shape[1]), + int(inputs.topk_indices.shape[1]), + q.device, + ) + inputs, slots_out, rows = self._exported_slots(inputs, metadata, module, image) result = direct_sparse_decode( query, inputs.swa_pool, @@ -499,20 +536,24 @@ def _forward_direct(self, q, metadata, forward_args): output=None if output is None else output.view_as(query), inputs_ready=ready, main_ready=main_ready, + plain_edge=plain_edge, query_rope=None if state is None else state.query_rope, + output_rope=state is not None and state.output_rope, slots_out=slots_out, + rows_out=rows if slots_out is not None else None, + rows_in=rows if main_ready else None, ) return result.flatten(1) - def _exported_slots(self, inputs, metadata, module): - """An index-source layer's direct decode stores the slots its mapped selections resolve to - (returned as ``slots_out``), and a Reuse layer whose source exported them this forward - attends those slots unmapped instead (``inputs`` updated); the layout gives both layers one - cache owner and compression ratio, so the same visible lengths.""" + def _exported_slots(self, inputs, metadata, module, image): + """An index source's direct decode exports the slots its mapped selections resolve to + (``slots_out``) and, with ``image`` bytes per query row, its decoded main rows (``rows``); a + Reuse layer of that source attends those slots unmapped and loads those rows. Both layers + have one cache owner and compression ratio, so the same visible lengths and main rows.""" layout = self.sparse_params.layout state = module.state if layout is None or state is None or inputs.main_mapping is None: - return inputs, None + return inputs, None, None source = state.metadata exported = source.csa2_direct_slots layer = layout.layer(self.layer_idx) @@ -520,16 +561,25 @@ def _exported_slots(self, inputs, metadata, module): if layer.mode == CSA2Mode.REUSE: slots = exported.get(layer.index_source) if slots is None or slots.shape[1] != inputs.topk_indices.shape[1]: - return inputs, None + return inputs, None, None inputs = replace(inputs, topk_indices=slots[start : start + count], main_mapping=None) - return inputs, None + rows = source.csa2_direct_rows.get(layer.index_source) + if rows is None or rows.shape[1] != image: + return inputs, None, None + return inputs, None, rows[start : start + count] slots = source._get_csa2_buffer( f"direct_slots/{self.layer_idx}", (state.swa_kv.shape[0], inputs.topk_indices.shape[1]), torch.int64, ) exported[self.layer_idx] = slots - return inputs, slots[start : start + count] + if not image: + return inputs, slots[start : start + count], None + rows = source._get_csa2_buffer( + f"direct_rows/{self.layer_idx}", (state.swa_kv.shape[0], image), torch.uint8 + ) + source.csa2_direct_rows[self.layer_idx] = rows + return inputs, slots[start : start + count], rows[start : start + count] def forward_fused_context(self, q, metadata, forward_args): """FP8 context attention of a shared-bank tile with one fused kernel. @@ -603,15 +653,62 @@ def prepare_sparse_inputs( or metadata.num_query_heads != self.num_heads ): raise ValueError("CSA2 Q geometry/dtype must match its BF16 metadata") - inputs = self._resolve_sparse_inputs(count, forward_args) + # SM100/SM103: the row write releases a direct decode early; it reads rows after its wait. + release = is_blackwell_datacenter() and self.direct_decode_applies( + q, metadata, forward_args + ) + inputs = self._resolve_sparse_inputs(count, forward_args, release_after_wait=release) if metadata.swa_pool.device != q.device: raise ValueError("CSA2 Q and metadata must be on the same CUDA device") return inputs + def publish_layer_rows( + self, state: CSA2ForwardState, *, release_after_wait: bool = False + ) -> CSA2ForwardState: + """Publish an entered layer's SWA (and a Full layer's global) rows in one quantize launch + that also packs an unpacked ``state.index_q``; returns ``state`` with packed queries.""" + source = state.metadata + layer = self.sparse_params.layout.layer(self.layer_idx) + global_rows = None + if layer.mode == CSA2Mode.FULL and self.layer_idx in source.csa2_precomputed_kv_layers: + if state.main_kv is not None or state.index_k is not None: + raise ValueError("CED precomputed global KV must not be projected twice") + elif layer.mode == CSA2Mode.FULL: + if state.main_kv is None or state.index_k is None: + raise ValueError("Full mode requires main and index rows, including empty rows") + global_rows = ( + layer.kv_source, + source.csa2_main_write_slots[layer.kv_source], + state.main_kv, + state.index_k, + state.main_transform, + state.index_transform, + ) + elif state.main_kv is not None or state.index_k is not None: + raise ValueError("Only Full mode may write shared main/index caches") + pack_queries = ( + layer.mode in (CSA2Mode.FULL, CSA2Mode.REINDEX) + and state.index_q is not None + and state.index_q_scale is None + ) + packed = source.kv_cache_manager.write_layer_rows( + self.layer_idx, + source.csa2_swa_write_slots[self.layer_idx], + state.swa_kv, + state.swa_transform, + global_rows=global_rows, + index_q=state.index_q if pack_queries else None, + release_after_wait=release_after_wait, + ) + if packed is not None: + state = replace(state, index_q=packed[0], index_q_scale=packed[1]) + return state + def _resolve_sparse_inputs( - self, count: int, forward_args: AttentionForwardArgs + self, count: int, forward_args: AttentionForwardArgs, *, release_after_wait: bool = False ) -> CSA2BackendForwardArgs: - """Publish a module tile's cache rows, run its indexer and return its selections.""" + """Publish a module tile's cache rows, run its indexer and return its selections; + ``release_after_wait``: the direct decode follows the rows (``prepare_sparse_inputs``).""" inputs = forward_args.sparse_backend_args if not isinstance(inputs, CSA2BackendForwardArgs): raise TypeError("CSA2 requires CSA2BackendForwardArgs") @@ -629,47 +726,15 @@ def _resolve_sparse_inputs( raise ValueError("CSA2 module inputs require the model layout") layer = layout.layer(self.layer_idx) start, end = inputs.query_start, inputs.query_start + count - if start == 0: + if start == 0 and not state.rows_published: source.enter_layer(layer) - global_rows = None - if ( - layer.mode == CSA2Mode.FULL - and self.layer_idx in source.csa2_precomputed_kv_layers - ): - if state.main_kv is not None or state.index_k is not None: - raise ValueError("CED precomputed global KV must not be projected twice") - elif layer.mode == CSA2Mode.FULL: - if state.main_kv is None or state.index_k is None: - raise ValueError( - "Full mode requires main and index rows, including empty rows" - ) - global_rows = ( - layer.kv_source, - source.csa2_main_write_slots[layer.kv_source], - state.main_kv, - state.index_k, - state.main_transform, - state.index_transform, - ) - elif state.main_kv is not None or state.index_k is not None: - raise ValueError("Only Full mode may write shared main/index caches") - # The SWA rows, the global rows and the index-query packing are - # one quantize launch; the indexer then consumes packed queries. - pack_queries = ( - layer.mode in (CSA2Mode.FULL, CSA2Mode.REINDEX) - and state.index_q is not None - and state.index_q_scale is None - ) - packed = manager.write_layer_rows( - self.layer_idx, - source.csa2_swa_write_slots[self.layer_idx], - state.swa_kv, - state.swa_transform, - global_rows=global_rows, - index_q=state.index_q if pack_queries else None, + # A SWA / Reuse layer writes only its SWA rows, always here (main_ready relies on + # it); a Full layer's also writes shared main / index rows and releases at its end. + state = self.publish_layer_rows( + state, + release_after_wait=release_after_wait + and layer.mode in (CSA2Mode.SWA, CSA2Mode.REUSE), ) - if packed is not None: - state = replace(state, index_q=packed[0], index_q_scale=packed[1]) logical = None if layer.mode in (CSA2Mode.FULL, CSA2Mode.REINDEX): if state.index_q is None or state.index_weights is None: @@ -758,6 +823,8 @@ def forward_fused_decode(self, q: torch.Tensor, forward_args: AttentionForwardAr table, page_size, max_positions, requests, visible = mapping or (None, 0, 0, None, None) if forward_args.sparse_backend_args.query_ready is not None: forward_args.sparse_backend_args.query_ready.wait() + if inputs.state.decode_start is not None: + inputs.state.decode_start.record() quant_q = inputs.projected_quant_q torch.ops.trtllm.csa2_sparse_decode( q if quant_q is None else quant_q.view(q.shape), diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py index 2dd0761239bd..2efce91ba950 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/cache_manager.py @@ -913,7 +913,15 @@ def write_global( store_index_rows(self.get_index_pages(layer_idx), slots, index, index_transform) def write_layer_rows( - self, layer_idx, slots, values, transform=None, *, global_rows=None, index_q=None + self, + layer_idx, + slots, + values, + transform=None, + *, + global_rows=None, + index_q=None, + release_after_wait=False, ): """One launch for a layer's SWA rows, its global rows and its packed index queries. @@ -937,6 +945,7 @@ def write_layer_rows( main=main, index=index, index_q=index_q, + release_after_wait=release_after_wait, ) def _get_page_index_converter(self, layer_id, role): diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/direct_decode.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/direct_decode.py index d54d2382027b..d12381570eb5 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/direct_decode.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/direct_decode.py @@ -23,15 +23,21 @@ sends each block with one bulk copy that completes on the owner's barrier (same values). The launch can also take the query before its RoPE and rotate it in shared memory, bit for bit -as a separate ``mla_rope_inplace`` launch does. An index-source layer's launch can export the -cache slots its mapped main selections resolve to, which a later launch over the same -selections, owner and visible lengths attends unmapped. +as a separate ``mla_rope_inplace`` launch does, and write its output rotated back by the same +angle, bit for bit as the small-M O-A GEMM rotates its BF16 input on load. An index-source +layer's launch can export the cache slots its mapped main selections resolve to, which a later +launch over the same selections, owner and visible lengths attends unmapped. Work that does not depend on the launch preceding this one on the stream happens before the programmatic dependent wait: the query and the SWA slots when ``inputs_ready`` promises they were final before that launch started, and the main rows (slots, page-table mapping, payload) -when ``main_ready`` promises the same for them; with both, the whole main part runs before -the wait and only the SWA rows the preceding cache write may have produced are left after it. +when ``main_ready`` promises they are final when this launch starts; with both, the whole main +part runs before the wait and only the SWA rows the preceding cache write may have produced are +left after it. With ``main_ready`` alone the main rows are loaded before the wait and decoded +after it, and the launch releases its own dependents only after the wait. + +At 8 query heads per rank (``_v2_engaged``) a CTA's key rows are the MMA's M and the heads its N, +with one softmax over the CTA's keys and an ``st.async`` merge; not bitwise to the two parts. """ import functools @@ -59,6 +65,12 @@ def direct_decode_device_supported(device: torch.device) -> bool: _SMEM_LIMIT = 227 * 1024 +@functools.cache +def _datacenter_blackwell(index: int) -> bool: + """SM100 / SM103, where this launch's optional variants run (not SM107).""" + return torch.cuda.get_device_capability(index) in ((10, 0), (10, 3)) + + def plan_split(swa_width: int, main_width: int) -> int: """CTAs per query row (one cluster) for a selection width.""" width = swa_width + main_width @@ -134,6 +146,82 @@ def smem_layout( } +def smem_layout_v2(keys: int) -> dict: + """Byte offsets of the keys-as-M attention's shared memory (``_v2_engaged``).""" + prow = 2 * keys + 16 + q = keys * _KEY_ROW + p = q + 8 * _KEY_ROW + red = pad_up(p + 8 * prow, 16) + fac = red + 2 * 8 * 8 * 4 + recv = pad_up(fac + 8 * 9 * 4, 128) + block = 4 * 32 * 16 + 8 * 8 + # Three mbarriers: the partial outputs', the (max, sum) pairs', then the imported main rows'. + mbar = recv + 8 * block + total_bytes = mbar + 24 + extras = keys * 12 + 16 * 20 + 6 * 128 + 64 * 4 + return { + "sp": 0, + "q": q, + "p": p, + "prow": prow, + "red": red, + "fac": fac, + "recv": recv, + "block": block, + "ml_off": 0, + "out": 0, + "mbar": mbar, + "bulk": False, + "bytes": total_bytes, + "total": total_bytes + extras, + "dedicated": True, + } + + +def _v2_engaged(q_heads: int, swa_width: int, main_width: int, device_index: int) -> bool: + """Whether a launch takes the keys-as-M attention instead of the two-part one.""" + if q_heads != 8 or not _datacenter_blackwell(device_index): + return False + cluster = plan_split(swa_width, main_width) + keys = sum(cta_keys(swa_width, main_width, cluster)) + return cluster == 8 and keys % 16 == 0 and 16 <= keys <= 128 + + +def main_rows_bytes(q_heads: int, swa_width: int, main_width: int, device: torch.device) -> int: + """Bytes per query row of the BF16 main-row image ``direct_sparse_decode`` stores + (``rows_out``) or loads (``rows_in``) for this geometry; 0 when the launch has none.""" + if not main_width or not _v2_engaged(q_heads, swa_width, main_width, device.index): + return 0 + cluster = plan_split(swa_width, main_width) + return cluster * cta_keys(swa_width, main_width, cluster)[0] * _KEY_ROW + + +# 58% (132 KB) shared-memory carveout: the keys-as-M CTA starts beside the projection GEMMs' CTAs. +_ROOM_CARVEOUT = 58 + + +def _set_smem_carveout(compiled, percent: int) -> None: + """Set the preferred shared-memory carveout of every kernel of a compiled CuTe DSL function on + the current device (the DSL's launch option does not reach the loaded kernels).""" + import cuda.bindings.driver as cuda + + err, device = cuda.cuCtxGetDevice() + if err != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"cuCtxGetDevice failed: {err}") + library = cuda.CUlibrary(int(compiled.library)) + for name in compiled.kernel_info: + err, kernel = cuda.cuLibraryGetKernel(library, name.encode()) + if err == cuda.CUresult.CUDA_SUCCESS: + (err,) = cuda.cuKernelSetAttribute( + cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, + percent, + kernel, + device, + ) + if err != cuda.CUresult.CUDA_SUCCESS: + raise RuntimeError(f"setting the shared-memory carveout of {name} failed: {err}") + + def mma_heads(heads: int) -> int: """Heads the kernel computes for ``heads`` query heads per rank (the ``mma.sync`` M dimension is 16 heads: 8 query heads, e.g. at TP8, fill one tile with 8 zero heads), 0 if unsupported.""" @@ -533,6 +621,57 @@ def main_decode8(w: Int32, scale: Float32, *, loc=None, ip=None): ) return words(out, 4, loc, ip) + @dsl_user_op + def swa_decode4_x2(w: Int32, scale: Float32, *, loc=None, ip=None): + """``swa_decode4`` with its scale multiplies paired as ``mul.rn.f32x2`` (same products).""" + out = asm( + llvm.StructType.get_literal([T.i32()] * 2), + [Int32(w).ir_value(loc=loc, ip=ip), Float32(scale).ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b16 lo, hi, x0, x1, x2, x3;\n\t.reg .b32 h0, h1;\n\t.reg .f32 f0, f1, f2, f3;\n\t" + ".reg .b64 s2, p0, p1;\n\t" + "mov.b32 {lo, hi}, $2;\n\tcvt.rn.f16x2.e4m3x2 h0, lo;\n\tcvt.rn.f16x2.e4m3x2 h1, hi;\n\t" + "mov.b32 {x0, x1}, h0;\n\tmov.b32 {x2, x3}, h1;\n\t" + "cvt.f32.f16 f0, x0;\n\tcvt.f32.f16 f1, x1;\n\tcvt.f32.f16 f2, x2;\n\tcvt.f32.f16 f3, x3;\n\t" + "mov.b64 s2, {$3, $3};\n\t" + "mov.b64 p0, {f0, f1};\n\tmul.rn.f32x2 p0, p0, s2;\n\tmov.b64 {f0, f1}, p0;\n\t" + "mov.b64 p1, {f2, f3};\n\tmul.rn.f32x2 p1, p1, s2;\n\tmov.b64 {f2, f3}, p1;\n\t" + "cvt.rn.bf16x2.f32 $0, f1, f0;\n\tcvt.rn.bf16x2.f32 $1, f3, f2;\n\t}", + "=r,=r,r,f", + side_effects=False, + loc=loc, + ip=ip, + ) + return words(out, 2, loc, ip) + + @dsl_user_op + def main_decode8_x2(w: Int32, scale: Float32, *, loc=None, ip=None): + """``main_decode8`` with its scale multiplies paired as ``mul.rn.f32x2`` (same products).""" + out = asm( + llvm.StructType.get_literal([T.i32()] * 4), + [Int32(w).ir_value(loc=loc, ip=ip), Float32(scale).ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b8 b0, b1, b2, b3;\n\t.reg .b16 x0, x1, x2, x3, x4, x5, x6, x7;\n\t" + ".reg .b32 h0, h1, h2, h3;\n\t.reg .f32 f0, f1, f2, f3, f4, f5, f6, f7;\n\t" + ".reg .b64 s2, p0, p1, p2, p3;\n\t" + "mov.b32 {b0, b1, b2, b3}, $4;\n\t" + "cvt.rn.f16x2.e2m1x2 h0, b0;\n\tcvt.rn.f16x2.e2m1x2 h1, b1;\n\t" + "cvt.rn.f16x2.e2m1x2 h2, b2;\n\tcvt.rn.f16x2.e2m1x2 h3, b3;\n\t" + "mov.b32 {x0, x1}, h0;\n\tmov.b32 {x2, x3}, h1;\n\tmov.b32 {x4, x5}, h2;\n\tmov.b32 {x6, x7}, h3;\n\t" + "cvt.f32.f16 f0, x0;\n\tcvt.f32.f16 f1, x1;\n\tcvt.f32.f16 f2, x2;\n\tcvt.f32.f16 f3, x3;\n\t" + "cvt.f32.f16 f4, x4;\n\tcvt.f32.f16 f5, x5;\n\tcvt.f32.f16 f6, x6;\n\tcvt.f32.f16 f7, x7;\n\t" + "mov.b64 s2, {$5, $5};\n\t" + "mov.b64 p0, {f0, f1};\n\tmul.rn.f32x2 p0, p0, s2;\n\tmov.b64 {f0, f1}, p0;\n\t" + "mov.b64 p1, {f2, f3};\n\tmul.rn.f32x2 p1, p1, s2;\n\tmov.b64 {f2, f3}, p1;\n\t" + "mov.b64 p2, {f4, f5};\n\tmul.rn.f32x2 p2, p2, s2;\n\tmov.b64 {f4, f5}, p2;\n\t" + "mov.b64 p3, {f6, f7};\n\tmul.rn.f32x2 p3, p3, s2;\n\tmov.b64 {f6, f7}, p3;\n\t" + "cvt.rn.bf16x2.f32 $0, f1, f0;\n\tcvt.rn.bf16x2.f32 $1, f3, f2;\n\t" + "cvt.rn.bf16x2.f32 $2, f5, f4;\n\tcvt.rn.bf16x2.f32 $3, f7, f6;\n\t}", + "=r,=r,=r,=r,r,f", + side_effects=False, + loc=loc, + ip=ip, + ) + return words(out, 4, loc, ip) + @dsl_user_op def bf16_pair(lo: Float32, hi: Float32, *, loc=None, ip=None) -> Int32: return Int32( @@ -591,6 +730,19 @@ def shfl_max(v: Float32, mask: int, *, loc=None, ip=None) -> Float32: ) ) + @dsl_user_op + def shfl_xor(v: Int32, mask: int, *, loc=None, ip=None) -> Int32: + return Int32( + asm( + T.i32(), + [Int32(v).ir_value(loc=loc, ip=ip)], + "shfl.sync.bfly.b32 $0, $1, " + str(mask) + ", 0x1f, 0xffffffff;", + "=r,r", + loc=loc, + ip=ip, + ) + ) + @dsl_user_op def shfl_sum(v: Float32, mask: int, *, loc=None, ip=None) -> Float32: return Float32( @@ -686,6 +838,20 @@ def ld_shared_f32x2(addr: Int32, *, loc=None, ip=None): ) return floats(out, 2, loc, ip) + @dsl_user_op + def st_shared_f32x4( + addr: Int32, a: Float32, b: Float32, c: Float32, d: Float32, *, loc=None, ip=None + ) -> None: + asm( + None, + [Int32(addr).ir_value(loc=loc, ip=ip)] + + [Float32(v).ir_value(loc=loc, ip=ip) for v in (a, b, c, d)], + "st.shared.v4.f32 [$0], {$1, $2, $3, $4};", + "r,f,f,f,f,~{memory}", + loc=loc, + ip=ip, + ) + @dsl_user_op def ld_shared_f32x4(addr: Int32, *, loc=None, ip=None): out = asm( @@ -698,6 +864,151 @@ def ld_shared_f32x4(addr: Int32, *, loc=None, ip=None): ) return floats(out, 4, loc, ip) + # ------------------------------------------------------- keys-as-M attention (v2) + + @dsl_user_op + def ldmatrix_x2(addr: Int32, *, loc=None, ip=None): + out = asm( + llvm.StructType.get_literal([T.i32()] * 2), + [Int32(addr).ir_value(loc=loc, ip=ip)], + "ldmatrix.sync.aligned.m8n8.x2.shared.b16 {$0, $1}, [$2];", + "=r,=r,r,~{memory}", + loc=loc, + ip=ip, + ) + return words(out, 2, loc, ip) + + @dsl_user_op + def shfl_idx8(v: Float32, k: int, *, loc=None, ip=None) -> Float32: + """Lane ``k`` of the caller's aligned group of 8 lanes.""" + return Float32( + asm( + T.f32(), + [Float32(v).ir_value(loc=loc, ip=ip)], + "shfl.sync.idx.b32 $0, $1, " + str(k) + ", 0x181f, 0xffffffff;", + "=f,f", + loc=loc, + ip=ip, + ) + ) + + @dsl_user_op + def st_async_f32x4( + addr: Int32, + mbar: Int32, + a: Float32, + b: Float32, + c: Float32, + d: Float32, + *, + loc=None, + ip=None, + ) -> None: + asm( + None, + [Int32(addr).ir_value(loc=loc, ip=ip), Int32(mbar).ir_value(loc=loc, ip=ip)] + + [Float32(v).ir_value(loc=loc, ip=ip) for v in (a, b, c, d)], + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [$0], {$2, $3, $4, $5}, [$1];", + "r,r,f,f,f,f,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def st_async_f32x2( + addr: Int32, mbar: Int32, a: Float32, b: Float32, *, loc=None, ip=None + ) -> None: + asm( + None, + [Int32(addr).ir_value(loc=loc, ip=ip), Int32(mbar).ir_value(loc=loc, ip=ip)] + + [Float32(v).ir_value(loc=loc, ip=ip) for v in (a, b)], + "st.async.shared::cluster.mbarrier::complete_tx::bytes.v2.f32 [$0], {$2, $3}, [$1];", + "r,r,f,f,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def mbar_init_expect(mbar: Int32, tx_bytes: int, *, loc=None, ip=None) -> None: + """One-arrival mbarrier expecting ``tx_bytes`` (its own arrive done), released to the + cluster (a cluster barrier must follow before others complete bytes on it).""" + asm( + None, + [Int32(mbar).ir_value(loc=loc, ip=ip)], + "mbarrier.init.shared::cta.b64 [$0], 1;\n\t" + f"mbarrier.arrive.expect_tx.shared::cta.b64 _, [$0], {tx_bytes};\n\t" + "fence.mbarrier_init.release.cluster;", + "r,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def mbar_wait0(mbar: Int32, *, loc=None, ip=None) -> None: + asm( + None, + [Int32(mbar).ir_value(loc=loc, ip=ip)], + "{\n\t.reg .pred p;\n\t" + "WAIT${:uid}:\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], 0;\n\t" + "@!p bra WAIT${:uid};\n\t}", + "r,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def cluster_wait_aligned(*, loc=None, ip=None) -> None: + asm(None, [], "barrier.cluster.wait.aligned;", "~{memory}", loc=loc, ip=ip) + + @dsl_user_op + def bulk_load_global(dst: Int32, src: Int64, size: int, mbar: Int32, *, loc=None, ip=None): + """Copy ``size`` bytes of global memory into this CTA's shared memory; the copy completes + its bytes on the barrier at ``mbar``.""" + asm( + None, + [ + Int32(dst).ir_value(loc=loc, ip=ip), + Int64(src).ir_value(loc=loc, ip=ip), + Int32(mbar).ir_value(loc=loc, ip=ip), + ], + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes " + f"[$0], [$1], {size}, [$2];", + "r,l,r,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def bulk_store_global(dst: Int64, src: Int32, size: int, *, loc=None, ip=None) -> None: + """Copy ``size`` bytes of this CTA's shared memory to global memory (``bulk_wait_read`` + before the CTA exits).""" + asm( + None, + [Int64(dst).ir_value(loc=loc, ip=ip), Int32(src).ir_value(loc=loc, ip=ip)], + f"cp.async.bulk.global.shared::cta.bulk_group [$0], [$1], {size};\n\t" + "cp.async.bulk.commit_group;", + "l,r,~{memory}", + loc=loc, + ip=ip, + ) + + @dsl_user_op + def bulk_wait_read(*, loc=None, ip=None) -> None: + """This thread's bulk copies have read their shared-memory sources.""" + asm(None, [], "cp.async.bulk.wait_group.read 0;", "~{memory}", loc=loc, ip=ip) + + @dsl_user_op + def st_global_b16(addr: Int64, w: Int32, *, loc=None, ip=None) -> None: + asm( + None, + [Int64(addr).ir_value(loc=loc, ip=ip), Int32(w).ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b16 h;\n\tcvt.u16.u32 h, $1;\n\tst.global.b16 [$0], h;\n\t}", + "l,r,~{memory}", + loc=loc, + ip=ip, + ) + # ------------------------------------------------- query RoPE, as mla_rope_inplace computes it @dsl_user_op @@ -769,12 +1080,20 @@ def __init__( inputs_ready: bool, main_ready: bool, use_pdl: bool, + plain_edge: bool, swa_vec: int = 16, main_vec: int = 8, q_heads: int = 0, q_rope: bool = False, bulk_exchange: bool = False, export_slots: bool = False, + fmul2: bool = False, + v2: bool = False, + main_prewait: bool = False, + o_rope: bool = False, + late_coefs: bool = False, + rows_out: bool = False, + rows_in: bool = False, ): self.q_heads = q_heads or heads self.heads = heads @@ -793,20 +1112,35 @@ def __init__( self.main_vec = main_vec self.inputs_ready = inputs_ready self.main_ready = main_ready and main_width > 0 + self.v2 = v2 # The main part runs before the dependent wait when its rows and the query are final. - self.pre_main = self.main_ready and inputs_ready + self.pre_main = self.main_ready and inputs_ready and not v2 + # Main rows final but the query not: they are loaded before the dependent wait. + self.main_prewait = main_prewait and use_pdl and self.main_ready and not self.pre_main self.use_pdl = use_pdl + self.plain_edge = plain_edge self.q_rope = q_rope + self.o_rope = o_rope + # The sinks and RoPE coefficients reach shared memory after the key resolve is issued. + self.late_coefs = late_coefs self.export_slots = export_slots and mapped and main_width > 0 - self.layout = smem_layout( - heads, - self.main_keys, - self.swa_keys, - cluster, - send_heads=min(self.q_heads, heads), - bulk=bulk_exchange, - ) + # Keys-as-M: ``rows_out`` stores the decoded main rows as laid out in shared memory; + # ``rows_in`` (main rows final at launch) copies that image in instead of decoding them. + self.rows_out = rows_out and v2 and not self.main_ready + self.rows_in = rows_in and v2 and self.main_ready + if v2: + self.layout = smem_layout_v2(self.keys) + else: + self.layout = smem_layout( + heads, + self.main_keys, + self.swa_keys, + cluster, + send_heads=min(self.q_heads, heads), + bulk=bulk_exchange, + ) self.bulk = self.layout["bulk"] + self.fmul2 = fmul2 @cute.jit def __call__( @@ -837,6 +1171,7 @@ def __call__( q_positions: cute.Pointer, q_cos_sin: cute.Pointer, slots_out: cute.Pointer, + main_rows: cute.Pointer, stream, ): self.kernel( @@ -865,13 +1200,14 @@ def __call__( q_positions, q_cos_sin, slots_out, + main_rows, ).launch( grid=[self.cluster, rows, 1], block=[_THREADS, 1, 1], cluster=[self.cluster, 1, 1], smem=self.layout["total"], stream=stream, - use_pdl=self.use_pdl, + use_pdl=self.use_pdl and not self.plain_edge, ) @cute.jit @@ -993,6 +1329,44 @@ def _decode_rows( data = cute.make_rmem_tensor(cute.make_layout((per_warp, 4), stride=(4, 1)), Int32) scale_byte = cute.make_rmem_tensor(cute.make_layout((per_warp,)), Int32) kinds = cute.make_rmem_tensor(cute.make_layout((per_warp,)), Int32) + self._load_rows( + warp, + lane, + swa, + main, + swa_stride, + main_stride, + sSlot, + sKind, + first, + count, + want, + data, + scale_byte, + kinds, + ) + self._store_rows(warp, lane, k_addr, first, count, want, data, scale_byte, kinds) + + @cute.jit + def _load_rows( + self, + warp, + lane, + swa, + main, + swa_stride, + main_stride, + sSlot, + sKind, + first: cutlass.Constexpr, + count: cutlass.Constexpr, + want: cutlass.Constexpr, + data, + scale_byte, + kinds, + ): + """The global loads of ``_decode_rows`` into registers, for ``_store_rows``.""" + per_warp: cutlass.Constexpr = ceil_div(count, 8) swa_base = Int64(swa.toint()) main_base = Int64(main.toint()) for i in cutlass.range_constexpr(per_warp): @@ -1039,26 +1413,59 @@ def _decode_rows( if const_expr(want & 4): if kind == 0: kinds[i] = Int32(0) + + @cute.jit + def _store_rows( + self, + warp, + lane, + k_addr, + first: cutlass.Constexpr, + count: cutlass.Constexpr, + want: cutlass.Constexpr, + data, + scale_byte, + kinds, + ): + """The rows ``_load_rows`` loaded, decoded into their shared-memory rows.""" + per_warp: cutlass.Constexpr = ceil_div(count, 8) for i in cutlass.range_constexpr(per_warp): r = first + warp + 8 * i dst = k_addr + r * _KEY_ROW + 32 * lane - if kinds[i] == 1: - s = swa_scale(scale_byte[i]) - o0, o1 = swa_decode4(data[i, 0], s) - o2, o3 = swa_decode4(data[i, 1], s) - o4, o5 = swa_decode4(data[i, 2], s) - o6, o7 = swa_decode4(data[i, 3], s) - st_shared_v4(dst, o0, o1, o2, o3) - st_shared_v4(dst + 16, o4, o5, o6, o7) - if kinds[i] == 2: - s = e4m3_scale(scale_byte[i]) - o0, o1, o2, o3 = main_decode8(data[i, 0], s) - o4, o5, o6, o7 = main_decode8(data[i, 1], s) - st_shared_v4(dst, o0, o1, o2, o3) - st_shared_v4(dst + 16, o4, o5, o6, o7) - if kinds[i] == 0: - st_shared_v4(dst, Int32(0), Int32(0), Int32(0), Int32(0)) - st_shared_v4(dst + 16, Int32(0), Int32(0), Int32(0), Int32(0)) + if const_expr(want & 1): + if kinds[i] == 1: + s = swa_scale(scale_byte[i]) + o0, o1 = self._swa4(data[i, 0], s) + o2, o3 = self._swa4(data[i, 1], s) + o4, o5 = self._swa4(data[i, 2], s) + o6, o7 = self._swa4(data[i, 3], s) + st_shared_v4(dst, o0, o1, o2, o3) + st_shared_v4(dst + 16, o4, o5, o6, o7) + if const_expr(want & 2): + if kinds[i] == 2: + s = e4m3_scale(scale_byte[i]) + o0, o1, o2, o3 = self._main8(data[i, 0], s) + o4, o5, o6, o7 = self._main8(data[i, 1], s) + st_shared_v4(dst, o0, o1, o2, o3) + st_shared_v4(dst + 16, o4, o5, o6, o7) + if const_expr(want & 4): + if kinds[i] == 0: + st_shared_v4(dst, Int32(0), Int32(0), Int32(0), Int32(0)) + st_shared_v4(dst + 16, Int32(0), Int32(0), Int32(0), Int32(0)) + + @cute.jit + def _main8(self, w, s): + if const_expr(self.fmul2): + return main_decode8_x2(w, s) + else: + return main_decode8(w, s) + + @cute.jit + def _swa4(self, w, s): + if const_expr(self.fmul2): + return swa_decode4_x2(w, s) + else: + return swa_decode4(w, s) @cute.jit def _part( @@ -1262,6 +1669,7 @@ def kernel( q_positions: cute.Pointer, q_cos_sin: cute.Pointer, slots_out: cute.Pointer, + main_rows: cute.Pointer, ): tidx, _, _ = cute.arch.thread_idx() rank, row, _ = cute.arch.block_idx() @@ -1308,15 +1716,53 @@ def kernel( if tidx == 0: mbarrier_init_tx(mbar_addr, Int32(cluster * block)) cluster_arrive_relaxed() - if const_expr(self.use_pdl): + if const_expr(self.v2): + # The partial outputs and the (max, sum) pairs complete on two mbarriers, initialized + # before any CTA stores (cluster barrier phase 1, waited before the push). + if tidx == 0: + mbar_init_expect(mbar_addr, 8 * (block - 64)) + mbar_init_expect(mbar_addr + 8, 8 * 64) + cluster_arrive_relaxed() + image: cutlass.Constexpr = main_keys * _KEY_ROW + image_src = Int64(main_rows.toint()) + Int64(row * cluster + rank) * Int64(image) + if const_expr(self.rows_in): + # The main rows' image streams in while the slots resolve. + if tidx == 0: + mbar_init_expect(mbar_addr + 16, image) + fence_proxy_async_shared() + bulk_load_global(k_addr, image_src, image, mbar_addr + 16) + if const_expr(self.use_pdl and not self.main_prewait): griddepcontrol_launch_dependents() q_heads: cutlass.Constexpr = self.q_heads q_base = Int64(q.toint()) + Int64(row) * Int64(q_heads * HEAD_DIM * 2) q_chunks: cutlass.Constexpr = q_heads * HEAD_DIM // 8 - if tidx < q_heads: - sSink[tidx] = ld_global_f32(Int64(sink.toint()) + Int64(4 * tidx)) * LOG2E - if const_expr(q_heads < heads): + # The sinks and the row's RoPE coefficients (its position is final long before this layer); + # with late_coefs through registers, so their loads never hold up the key rows' resolve. + sink_v = Float32(0.0) + rope_pos = Int32(0) + rope_v = Float32(0.0) + if const_expr(self.late_coefs): + if tidx < q_heads: + sink_v = ld_global_f32(Int64(sink.toint()) + Int64(4 * tidx)) + if const_expr(self.q_rope): + if tidx < 64: + rope_pos = ld_global_s32(Int64(q_positions.toint()) + Int64(row) * 4) + else: + if tidx < q_heads: + sSink[tidx] = ld_global_f32(Int64(sink.toint()) + Int64(4 * tidx)) * LOG2E + if const_expr(self.q_rope): + if tidx < 64: + sRope[tidx] = ld_global_f32( + Int64(q_cos_sin.toint()) + + Int64(ld_global_s32(Int64(q_positions.toint()) + Int64(row) * 4)) + * Int64(64 * 4) + + Int64(4 * tidx) + ) + # Main rows resolved before the dependent wait: the coefficients load right after them. + rope_early: cutlass.Constexpr = self.pre_main or self.main_prewait + late_rope: cutlass.Constexpr = self.late_coefs and self.q_rope + if const_expr(q_heads < heads and not self.v2): # Padding heads: a zero query row (their outputs are not stored). if tidx >= q_heads: if tidx < heads: @@ -1330,21 +1776,20 @@ def kernel( Int32(0), Int32(0), ) - oacc0 = cute.make_rmem_tensor(cute.make_layout((o_tiles, 4), stride=(4, 1)), Float32) - oacc1 = cute.make_rmem_tensor(cute.make_layout((o_tiles, 4), stride=(4, 1)), Float32) - oacc0.fill(0.0) - oacc1.fill(0.0) - # The row's RoPE coefficients (its position is final long before this layer). - if const_expr(self.q_rope): - if tidx < 64: - sRope[tidx] = ld_global_f32( - Int64(q_cos_sin.toint()) - + Int64(ld_global_s32(Int64(q_positions.toint()) + Int64(row) * 4)) - * Int64(64 * 4) - + Int64(4 * tidx) - ) + oacc0 = oacc1 = None + if const_expr(not self.v2): + oacc0 = cute.make_rmem_tensor( + cute.make_layout((o_tiles, 4), stride=(4, 1)), Float32 + ) + oacc1 = cute.make_rmem_tensor( + cute.make_layout((o_tiles, 4), stride=(4, 1)), Float32 + ) + oacc0.fill(0.0) + oacc1.fill(0.0) # ---- before the dependent wait: inputs final before the preceding launch ---- + # The SWA slots are final whenever the main rows are promised before the query. + swa_first: cutlass.Constexpr = self.inputs_ready or self.main_prewait if const_expr(self.inputs_ready): for i in cutlass.range_constexpr(q_chunks // _THREADS): c = tidx + _THREADS * i @@ -1372,12 +1817,41 @@ def kernel( max_positions, swa_capacity, main_capacity, - self.inputs_ready, + swa_first, self.main_ready, sSlot, sKind, slots_out, ) + if const_expr(late_rope and rope_early): + if tidx < 64: + rope_v = ld_global_f32( + Int64(q_cos_sin.toint()) + Int64(rope_pos) * (64 * 4) + Int64(4 * tidx) + ) + # main_prewait: the main rows are loaded before the dependent wait and decoded after it. + main_loads: cutlass.Constexpr = self.main_prewait and not self.rows_in + m_rows: cutlass.Constexpr = ceil_div(main_keys, 8) if main_loads else 1 + m_data = cute.make_rmem_tensor(cute.make_layout((m_rows, 4), stride=(4, 1)), Int32) + m_scale = cute.make_rmem_tensor(cute.make_layout((m_rows,)), Int32) + m_kinds = cute.make_rmem_tensor(cute.make_layout((m_rows,)), Int32) + if const_expr(main_loads): + cute.arch.sync_threads() + self._load_rows( + warp, + lane, + swa, + main, + swa_stride, + main_stride, + sSlot, + sKind, + 0, + main_keys, + 6, + m_data, + m_scale, + m_kinds, + ) if const_expr(self.pre_main): cute.arch.sync_threads() self._decode_rows( @@ -1394,6 +1868,8 @@ def kernel( main_keys, 6, ) + if const_expr(self.late_coefs): + self._store_coefs(tidx, sSink, sRope, sink_v, rope_v) cp_async_wait_all() cute.arch.sync_threads() if const_expr(self.q_rope): @@ -1418,6 +1894,9 @@ def kernel( if const_expr(self.use_pdl): griddepcontrol_wait() + if const_expr(self.main_prewait): + # Started under its producers: the next projection launches once they completed. + griddepcontrol_launch_dependents() # ---- after it: this layer's cache rows (and anything not promised ready) ---- if const_expr(not self.inputs_ready): @@ -1426,7 +1905,7 @@ def kernel( cp_async16( q_addr + (c // 64) * _KEY_ROW + (c % 64) * 16, q_base + Int64(16 * c) ) - if const_expr((not self.inputs_ready) or (not self.main_ready)): + if const_expr((not swa_first) or (not self.main_ready)): if tidx < keys: self._resolve( tidx, @@ -1446,14 +1925,60 @@ def kernel( max_positions, swa_capacity, main_capacity, - not self.inputs_ready, + not swa_first, not self.main_ready, sSlot, sKind, slots_out, ) + if const_expr(late_rope and not rope_early): + if tidx < 64: + rope_v = ld_global_f32( + Int64(q_cos_sin.toint()) + Int64(rope_pos) * (64 * 4) + Int64(4 * tidx) + ) cute.arch.sync_threads() - if const_expr(self.pre_main): + if const_expr(self.rows_in): + self._decode_rows( + warp, + lane, + k_addr, + swa, + main, + swa_stride, + main_stride, + sSlot, + sKind, + main_keys, + swa_keys, + 5, + ) + elif const_expr(self.main_prewait): + # This layer's SWA rows go out first; the main rows decode while they are in flight. + s_rows: cutlass.Constexpr = ceil_div(swa_keys, 8) + s_data = cute.make_rmem_tensor(cute.make_layout((s_rows, 4), stride=(4, 1)), Int32) + s_scale = cute.make_rmem_tensor(cute.make_layout((s_rows,)), Int32) + s_kinds = cute.make_rmem_tensor(cute.make_layout((s_rows,)), Int32) + self._load_rows( + warp, + lane, + swa, + main, + swa_stride, + main_stride, + sSlot, + sKind, + main_keys, + swa_keys, + 5, + s_data, + s_scale, + s_kinds, + ) + self._store_rows(warp, lane, k_addr, 0, main_keys, 6, m_data, m_scale, m_kinds) + self._store_rows( + warp, lane, k_addr, main_keys, swa_keys, 5, s_data, s_scale, s_kinds + ) + elif const_expr(self.pre_main): self._decode_rows( warp, lane, @@ -1483,166 +2008,507 @@ def kernel( keys, 7, ) + if const_expr(self.late_coefs and not self.pre_main): + self._store_coefs(tidx, sSink, sRope, sink_v, rope_v) + if const_expr(self.rows_out): + # The main rows leave through the async proxy (one bulk copy, behind this sync). + fence_proxy_async_shared() cp_async_wait_all() + if const_expr(self.rows_in): + mbar_wait0(mbar_addr + 16) cute.arch.sync_threads() + if const_expr(self.rows_out): + if tidx == 0: + bulk_store_global(image_src, k_addr, image) if const_expr(self.q_rope and not self.pre_main): self._rope_query(tidx, q_addr, sRope) cute.arch.sync_threads() - if const_expr((not self.pre_main) and main_keys > 0): - self._part( + if const_expr(self.v2): + self._v2_core( tidx, warp, lane, - q_addr, + rank, + row, k_addr, - sp_addr, - p_addr, - sKind, - sML, - oacc0, - scale_log2, - 0, - main_keys, - 0, - ) - if const_expr(swa_keys > 0): - self._part( - tidx, - warp, - lane, q_addr, - k_addr, - sp_addr, - p_addr, + base, sKind, - sML, - oacc1, + sSink, + sRope, + out, scale_log2, - main_keys, - swa_keys, - 1, ) - cute.arch.sync_threads() - - # ---- merge the parts, then send this CTA's partial to the channel owners ---- - ht = warp % head_tiles - ds = warp // head_tiles - # This CTA's block in every destination's receive buffer. - send_base = recv_addr + rank * block - # Each thread holds fragment rows ``g`` and ``g + 8``; with 8 query heads the second are - # padding. - halves: cutlass.Constexpr = 1 if q_heads < heads else 2 - for half in cutlass.range_constexpr(halves): - h = 16 * ht + g + 8 * half - f0 = Float32(0.0) - f1 = Float32(0.0) - if const_expr(main_keys > 0 and swa_keys > 0): - mm0 = sML[2 * h] - mm1 = sML[2 * heads + 2 * h] - mtop = cute.arch.fmax(mm0, mm1) - if mm0 != Float32(NEG_INF): - f0 = exp2(mm0 - mtop) - if mm1 != Float32(NEG_INF): - f1 = exp2(mm1 - mtop) - elif const_expr(main_keys > 0): - f0 = Float32(1.0) - else: - f1 = Float32(1.0) - for n in cutlass.range_constexpr(o_tiles): - d = ds * d_slice + 8 * n + 2 * t - v0 = oacc0[n, 2 * half] * f0 + oacc1[n, 2 * half] * f1 - v1 = oacc0[n, 2 * half + 1] * f0 + oacc1[n, 2 * half + 1] * f1 - if const_expr(self.bulk): - dest, offset = self._channel_owner(h, d) - st_shared_f32x2(out_addr + dest * block + offset * 4, v0, v1) - elif const_expr(lay["dedicated"]): - dest, offset = self._channel_owner(h, d) - st_cluster_f32x2(mapa(send_base + offset * 4, dest), v0, v1) - else: - oacc0[n, 2 * half] = v0 - oacc0[n, 2 * half + 1] = v1 - # This CTA's (max, sum) per head over both parts, sent with its partial outputs. - ml_top = Float32(NEG_INF) - ml_total = Float32(0.0) - pm0 = Float32(NEG_INF) - pl0 = Float32(0.0) - pm1 = Float32(NEG_INF) - pl1 = Float32(0.0) - if tidx < q_heads: - if const_expr(main_keys > 0): - pm0 = sML[2 * tidx] - pl0 = sML[2 * tidx + 1] + if const_expr(self.rows_out): + if tidx == 0: + bulk_wait_read() + else: + if const_expr((not self.pre_main) and main_keys > 0): + self._part( + tidx, + warp, + lane, + q_addr, + k_addr, + sp_addr, + p_addr, + sKind, + sML, + oacc0, + scale_log2, + 0, + main_keys, + 0, + ) if const_expr(swa_keys > 0): - pm1 = sML[2 * heads + 2 * tidx] - pl1 = sML[2 * heads + 2 * tidx + 1] - ml_top = cute.arch.fmax(pm0, pm1) - if pm0 != Float32(NEG_INF): - ml_total = ml_total + pl0 * exp2(pm0 - ml_top) - if pm1 != Float32(NEG_INF): - ml_total = ml_total + pl1 * exp2(pm1 - ml_top) - if const_expr(self.bulk): - for dest in cutlass.range_constexpr(cluster): - st_shared_f32x2( - out_addr + dest * block + ml_off + tidx * 8, ml_top, ml_total - ) - elif const_expr(lay["dedicated"]): - for dest in cutlass.range_constexpr(cluster): - st_cluster_f32x2( - mapa(send_base + ml_off + tidx * 8, dest), ml_top, ml_total - ) - if const_expr(not lay["dedicated"]): - # The receive buffer reuses the key rows: every CTA must be done with them. - cute.arch.cluster_arrive() - cute.arch.cluster_wait() + self._part( + tidx, + warp, + lane, + q_addr, + k_addr, + sp_addr, + p_addr, + sKind, + sML, + oacc1, + scale_log2, + main_keys, + swa_keys, + 1, + ) + cute.arch.sync_threads() + + # ---- merge the parts, then send this CTA's partial to the channel owners ---- + ht = warp % head_tiles + ds = warp // head_tiles + # This CTA's block in every destination's receive buffer. + send_base = recv_addr + rank * block + # Each thread holds fragment rows ``g`` and ``g + 8`` (padding at 8 query heads). + halves: cutlass.Constexpr = 1 if q_heads < heads else 2 for half in cutlass.range_constexpr(halves): h = 16 * ht + g + 8 * half + f0 = Float32(0.0) + f1 = Float32(0.0) + if const_expr(main_keys > 0 and swa_keys > 0): + mm0 = sML[2 * h] + mm1 = sML[2 * heads + 2 * h] + mtop = cute.arch.fmax(mm0, mm1) + if mm0 != Float32(NEG_INF): + f0 = exp2(mm0 - mtop) + if mm1 != Float32(NEG_INF): + f1 = exp2(mm1 - mtop) + elif const_expr(main_keys > 0): + f0 = Float32(1.0) + else: + f1 = Float32(1.0) for n in cutlass.range_constexpr(o_tiles): d = ds * d_slice + 8 * n + 2 * t - dest, offset = self._channel_owner(h, d) - st_cluster_f32x2( - mapa(send_base + offset * 4, dest), - oacc0[n, 2 * half], - oacc0[n, 2 * half + 1], - ) + v0 = oacc0[n, 2 * half] * f0 + oacc1[n, 2 * half] * f1 + v1 = oacc0[n, 2 * half + 1] * f0 + oacc1[n, 2 * half + 1] * f1 + if const_expr(self.bulk): + dest, offset = self._channel_owner(h, d) + st_shared_f32x2(out_addr + dest * block + offset * 4, v0, v1) + elif const_expr(lay["dedicated"]): + dest, offset = self._channel_owner(h, d) + st_cluster_f32x2(mapa(send_base + offset * 4, dest), v0, v1) + else: + oacc0[n, 2 * half] = v0 + oacc0[n, 2 * half + 1] = v1 + # This CTA's (max, sum) per head over both parts, sent with its partial outputs. + ml_top = Float32(NEG_INF) + ml_total = Float32(0.0) + pm0 = Float32(NEG_INF) + pl0 = Float32(0.0) + pm1 = Float32(NEG_INF) + pl1 = Float32(0.0) if tidx < q_heads: - for dest in cutlass.range_constexpr(cluster): - st_cluster_f32x2( - mapa(send_base + ml_off + tidx * 8, dest), ml_top, ml_total + if const_expr(main_keys > 0): + pm0 = sML[2 * tidx] + pl0 = sML[2 * tidx + 1] + if const_expr(swa_keys > 0): + pm1 = sML[2 * heads + 2 * tidx] + pl1 = sML[2 * heads + 2 * tidx + 1] + ml_top = cute.arch.fmax(pm0, pm1) + if pm0 != Float32(NEG_INF): + ml_total = ml_total + pl0 * exp2(pm0 - ml_top) + if pm1 != Float32(NEG_INF): + ml_total = ml_total + pl1 * exp2(pm1 - ml_top) + if const_expr(self.bulk): + for dest in cutlass.range_constexpr(cluster): + st_shared_f32x2( + out_addr + dest * block + ml_off + tidx * 8, ml_top, ml_total + ) + elif const_expr(lay["dedicated"]): + for dest in cutlass.range_constexpr(cluster): + st_cluster_f32x2( + mapa(send_base + ml_off + tidx * 8, dest), ml_top, ml_total + ) + if const_expr(not lay["dedicated"]): + # The receive buffer reuses the key rows: every CTA must be done with them. + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + for half in cutlass.range_constexpr(halves): + h = 16 * ht + g + 8 * half + for n in cutlass.range_constexpr(o_tiles): + d = ds * d_slice + 8 * n + 2 * t + dest, offset = self._channel_owner(h, d) + st_cluster_f32x2( + mapa(send_base + offset * 4, dest), + oacc0[n, 2 * half], + oacc0[n, 2 * half + 1], + ) + if tidx < q_heads: + for dest in cutlass.range_constexpr(cluster): + st_cluster_f32x2( + mapa(send_base + ml_off + tidx * 8, dest), ml_top, ml_total + ) + if const_expr(self.bulk): + # Each destination's block goes as one copy; every source's block is visible + # once this CTA's barrier completes its bytes. + fence_proxy_async_shared() + cute.arch.sync_threads() + cute.arch.cluster_wait() + if tidx < cluster: + bulk_copy_cluster( + mapa(recv_addr + rank * block, tidx), + out_addr + tidx * block, + Int32(block), + mapa(mbar_addr, tidx), ) - if const_expr(self.bulk): - # Each destination's block goes as one copy; every source's block is visible once - # this CTA's barrier completes its bytes. - fence_proxy_async_shared() + mbarrier_wait_parity(mbar_addr, Int32(0)) + # The copies complete only on their destinations' barriers: no CTA may leave + # while a copy still reads its shared memory (the cluster wait at the end). + cluster_arrive_relaxed() + else: + # Every source's block (and its arrive's release) is visible after the wait. + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + + # ---- this CTA's channels of every head: sum the sources with the sink ---- + items: cutlass.Constexpr = q_heads * share // 4 + out_base = Int64(out.toint()) + Int64(row) * Int64(q_heads * HEAD_DIM * 2) + for i in cutlass.range_constexpr(ceil_div(items, _THREADS)): + item = tidx + _THREADS * i + if item < items: + h = item // (share // 4) + dd = 4 * (item % (share // 4)) + w01, w23 = self._merge4(recv_addr, sSink, h, h * share + dd) + if const_expr(self.o_rope): + channel = rank * share + dd + if channel >= HEAD_DIM - 64: + pair = (channel - (HEAD_DIM - 64)) // 2 + w01 = self._rope_back(sRope, pair, w01) + w23 = self._rope_back(sRope, pair + 1, w23) + st_global_v2( + out_base + Int64((h * HEAD_DIM + rank * share + dd) * 2), w01, w23 + ) + if const_expr(self.bulk): + cute.arch.cluster_wait() + + @cute.jit + def _store_coefs(self, tidx, sSink, sRope, sink_v, rope_v): + """The sinks (log2 units) and the query RoPE coefficients into shared memory.""" + if tidx < self.q_heads: + sSink[tidx] = sink_v * LOG2E + if const_expr(self.q_rope): + if tidx < 64: + sRope[tidx] = rope_v + + @cute.jit + def _s_chain(self, acc, slot: cutlass.Constexpr, a_row, b_row, chain): + """Accumulator chain ``chain`` of a key tile's S^T (k steps ``chain + 4i``, in order) + into ``acc[slot]``.""" + for i in cutlass.range_constexpr(HEAD_DIM // 64): + j = chain + 4 * i + qa0, qa1, qa2, qa3 = ldmatrix_x4(a_row + 32 * j, False) + qb0, qb1 = ldmatrix_x2(b_row + 32 * j) + r0, r1, r2, r3 = mma_bf16( + qa0, + qa1, + qa2, + qa3, + qb0, + qb1, + acc[slot, 0], + acc[slot, 1], + acc[slot, 2], + acc[slot, 3], + ) + acc[slot, 0] = r0 + acc[slot, 1] = r1 + acc[slot, 2] = r2 + acc[slot, 3] = r3 + + @cute.jit + def _rope_back(self, sRope, pair, w): + """A BF16 channel pair rotated back by its RoPE angle as the small-M O-A GEMM rotates + its input on load (FP32, one BF16 rounding).""" + x, y = bf16_halves(w) + c = sRope[pair] + s = sRope[32 + pair] + return bf16_pair(fma_rn(x, c, mul_rn(y, s)), fma_rn(y, c, -mul_rn(x, s))) + + @cute.jit + def _rope_back_lanes(self, sRope, pair, odd, wa, wb): + """``_rope_back`` of one channel of two heads (BF16 ``wa``, ``wb``) whose pair partners + are lane ``^ 4``'s.""" + own = wa | (wb << 16) + xa, xb = bf16_halves(own) + pa, pb = bf16_halves(shfl_xor(own, 4)) + c = sRope[pair] + s = sRope[32 + pair] + if odd == 1: + s = -s + return bf16_bits(fma_rn(xa, c, mul_rn(pa, s))), bf16_bits(fma_rn(xb, c, mul_rn(pb, s))) + + @cute.jit + def _v2_core( + self, + tidx, + warp, + lane, + rank, + row, + k_addr, + q_addr, + base, + sKind, + sSink, + sRope, + out, + scale_log2, + ): + """Keys-as-M attention of the 8 query heads over this CTA's key rows, then the merge.""" + kt: cutlass.Constexpr = self.keys // 16 + lay = self.layout + prow: cutlass.Constexpr = lay["prow"] + block: cutlass.Constexpr = lay["block"] + p_addr = base + lay["p"] + red_max = base + lay["red"] + red_sum = red_max + 8 * 8 * 4 + recv_addr = base + lay["recv"] + g = lane // 4 + t = lane % 4 + key_lo = 16 * warp + g + key_hi = key_lo + 8 + # ---- S^T for this warp's 16 key rows (four accumulator chains) ---- + v0 = Float32(NEG_INF) + v1 = Float32(NEG_INF) + v2 = Float32(NEG_INF) + v3 = Float32(NEG_INF) + acc = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), Float32) + acc.fill(0.0) + a_lane = (lane % 16) * _KEY_ROW + 16 * (lane // 16) + b_row = q_addr + (lane % 8) * _KEY_ROW + 16 * ((lane // 8) % 2) + if const_expr(kt <= 2): + # Few tiles: chain c > 0 of tile t (v = 3t + c - 1) runs on warp (kt + v) % 8 and + # passes its partial through the query rows once every MMA has read them. + if warp < kt: + self._s_chain(acc, 0, k_addr + 16 * warp * _KEY_ROW + a_lane, b_row, 0) + v = (warp + 8 - kt) % 8 + if v < 3 * kt: + self._s_chain( + acc, 1, k_addr + 16 * (v // 3) * _KEY_ROW + a_lane, b_row, 1 + v % 3 + ) cute.arch.sync_threads() - cute.arch.cluster_wait() - if tidx < cluster: - bulk_copy_cluster( - mapa(recv_addr + rank * block, tidx), - out_addr + tidx * block, - Int32(block), - mapa(mbar_addr, tidx), + if v < 3 * kt: + st_shared_f32x4( + q_addr + v * 512 + lane * 16, acc[1, 0], acc[1, 1], acc[1, 2], acc[1, 3] ) - mbarrier_wait_parity(mbar_addr, Int32(0)) - # The copies complete only on their destinations' barriers: no CTA may leave while - # a copy still reads its shared memory (the cluster wait at the end). - cluster_arrive_relaxed() - else: - # Every source's block (and its arrive's release) is visible after the wait. - cute.arch.cluster_arrive() - cute.arch.cluster_wait() - - # ---- this CTA's channels of every head: sum the sources with the sink ---- - items: cutlass.Constexpr = q_heads * share // 4 - out_base = Int64(out.toint()) + Int64(row) * Int64(q_heads * HEAD_DIM * 2) - for i in cutlass.range_constexpr(ceil_div(items, _THREADS)): - item = tidx + _THREADS * i - if item < items: - h = item // (share // 4) - dd = 4 * (item % (share // 4)) - w01, w23 = self._merge4(recv_addr, sSink, h, h * share + dd) - st_global_v2(out_base + Int64((h * HEAD_DIM + rank * share + dd) * 2), w01, w23) - if const_expr(self.bulk): - cute.arch.cluster_wait() + cute.arch.sync_threads() + if warp < kt: + for c in cutlass.range_constexpr(1, 4): + p0, p1, p2, p3 = ld_shared_f32x4( + q_addr + (3 * warp + c - 1) * 512 + lane * 16 + ) + acc[c, 0] = p0 + acc[c, 1] = p1 + acc[c, 2] = p2 + acc[c, 3] = p3 + elif warp < kt: + a_row = k_addr + 16 * warp * _KEY_ROW + a_lane + for j in cutlass.range_constexpr(HEAD_DIM // 16): + qa0, qa1, qa2, qa3 = ldmatrix_x4(a_row + 32 * j, False) + qb0, qb1 = ldmatrix_x2(b_row + 32 * j) + c = j % 4 + r0, r1, r2, r3 = mma_bf16( + qa0, qa1, qa2, qa3, qb0, qb1, acc[c, 0], acc[c, 1], acc[c, 2], acc[c, 3] + ) + acc[c, 0] = r0 + acc[c, 1] = r1 + acc[c, 2] = r2 + acc[c, 3] = r3 + if warp < kt: + v0 = ((acc[0, 0] + acc[1, 0]) + (acc[2, 0] + acc[3, 0])) * scale_log2 + v1 = ((acc[0, 1] + acc[1, 1]) + (acc[2, 1] + acc[3, 1])) * scale_log2 + v2 = ((acc[0, 2] + acc[1, 2]) + (acc[2, 2] + acc[3, 2])) * scale_log2 + v3 = ((acc[0, 3] + acc[1, 3]) + (acc[2, 3] + acc[3, 3])) * scale_log2 + if sKind[key_lo] == 0: + v0 = Float32(NEG_INF) + v1 = Float32(NEG_INF) + if sKind[key_hi] == 0: + v2 = Float32(NEG_INF) + v3 = Float32(NEG_INF) + wm_a = cute.arch.fmax(v0, v2) + wm_b = cute.arch.fmax(v1, v3) + for mask in (4, 8, 16): + wm_a = shfl_max(wm_a, mask) + wm_b = shfl_max(wm_b, mask) + if g == 0: + st_shared_f32(red_max + (warp * 8 + 2 * t) * 4, wm_a) + st_shared_f32(red_max + (warp * 8 + 2 * t + 1) * 4, wm_b) + cute.arch.sync_threads() + # ---- numerators against the CTA's maximum per head; BF16 probabilities ---- + if warp < kt: + cm_a = Float32(NEG_INF) + cm_b = Float32(NEG_INF) + for w in cutlass.range_constexpr(kt): + cm_a = cute.arch.fmax(cm_a, ld_shared_f32(red_max + (w * 8 + 2 * t) * 4)) + cm_b = cute.arch.fmax(cm_b, ld_shared_f32(red_max + (w * 8 + 2 * t + 1) * 4)) + if cm_a == Float32(NEG_INF): + cm_a = Float32(0.0) + if cm_b == Float32(NEG_INF): + cm_b = Float32(0.0) + p0 = exp2(v0 - cm_a) + p1 = exp2(v1 - cm_b) + p2 = exp2(v2 - cm_a) + p3 = exp2(v3 - cm_b) + st_shared_b16(p_addr + (2 * t) * prow + 2 * key_lo, bf16_bits(p0)) + st_shared_b16(p_addr + (2 * t + 1) * prow + 2 * key_lo, bf16_bits(p1)) + st_shared_b16(p_addr + (2 * t) * prow + 2 * key_hi, bf16_bits(p2)) + st_shared_b16(p_addr + (2 * t + 1) * prow + 2 * key_hi, bf16_bits(p3)) + ps_a = p0 + p2 + ps_b = p1 + p3 + for mask in (4, 8, 16): + ps_a = shfl_sum(ps_a, mask) + ps_b = shfl_sum(ps_b, mask) + if g == 0: + st_shared_f32(red_sum + (warp * 8 + 2 * t) * 4, ps_a) + st_shared_f32(red_sum + (warp * 8 + 2 * t + 1) * 4, ps_b) + cute.arch.sync_threads() + # Every CTA's mbarrier is initialized (cluster barrier phase 1): this CTA's (max, sum) per + # head and then its partial outputs go out as soon as each is final. + mbar = base + lay["mbar"] + send = recv_addr + rank * block + cluster_wait_aligned() + if tidx < 8: + hm = Float32(NEG_INF) + hl = Float32(0.0) + for w in cutlass.range_constexpr(kt): + hm = cute.arch.fmax(hm, ld_shared_f32(red_max + (w * 8 + tidx) * 4)) + hl = hl + ld_shared_f32(red_sum + (w * 8 + tidx) * 4) + for dest in cutlass.range_constexpr(8): + st_async_f32x2( + mapa(send + 4 * 32 * 16 + tidx * 8, dest), mapa(mbar + 8, dest), hm, hl + ) + # ---- O^T for channels 64w .. 64w + 63 over every key row, to their owner (rank w) ---- + oc = cute.make_rmem_tensor(cute.make_layout((4, 4), stride=(4, 1)), Float32) + oc.fill(0.0) + pb = cute.make_rmem_tensor(cute.make_layout((kt, 2), stride=(2, 1)), Int32) + pb_row = p_addr + (lane % 8) * prow + 16 * ((lane // 8) % 2) + v_row = k_addr + (lane % 8 + 8 * (lane // 16)) * _KEY_ROW + 16 * ((lane // 8) % 2) + for ks in cutlass.range_constexpr(kt): + pb0, pb1 = ldmatrix_x2(pb_row + 32 * ks) + pb[ks, 0] = pb0 + pb[ks, 1] = pb1 + # In two halves of 32 channels: the first half's bytes travel under the second's MMAs. + for half in cutlass.range_constexpr(2): + for ks in cutlass.range_constexpr(kt): + for i in cutlass.range_constexpr(2 * half, 2 * half + 2): + pa0, pa1, pa2, pa3 = ldmatrix_x4( + v_row + (16 * ks) * _KEY_ROW + 2 * (64 * warp + 16 * i), True + ) + c0, c1, c2, c3 = mma_bf16( + pa0, + pa1, + pa2, + pa3, + pb[ks, 0], + pb[ks, 1], + oc[i, 0], + oc[i, 1], + oc[i, 2], + oc[i, 3], + ) + oc[i, 0] = c0 + oc[i, 1] = c1 + oc[i, 2] = c2 + oc[i, 3] = c3 + for i in cutlass.range_constexpr(2 * half, 2 * half + 2): + st_async_f32x4( + mapa(send + (i * 32 + lane) * 16, warp), + mapa(mbar, warp), + oc[i, 0], + oc[i, 1], + oc[i, 2], + oc[i, 3], + ) + # Phase 2 (waited at the end): no CTA exits while another may still store into it. + cluster_arrive_relaxed() + # ---- per head: the sources' weights against the cluster maximum (with the sink) ---- + mbar_wait0(mbar + 8) + fac = base + lay["fac"] + if tidx < 64: + # A head's 8 sources are one lane octet. + fh = tidx // 8 + fs_src = tidx % 8 + sink_h = sSink[fh] + rm, rl = ld_shared_f32x2(recv_addr + fs_src * block + 4 * 32 * 16 + fh * 8) + top = cute.arch.fmax(sink_h, rm) + for mask in (1, 2, 4): + top = shfl_max(top, mask) + fs = Float32(0.0) + if rm != Float32(NEG_INF): + fs = exp2(rm - top) + st_shared_f32(fac + (fh * 9 + fs_src) * 4, fs) + part = fs * rl + den = exp2(sink_h - top) + for k in cutlass.range_constexpr(8): + den = den + shfl_idx8(part, k) + if fs_src == 0: + st_shared_f32(fac + (fh * 9 + 8) * 4, Float32(1.0) / den) + cute.arch.sync_threads() + mbar_wait0(mbar) + # ---- this CTA's channels 64 rank .. 64 rank + 63 of every head ---- + if tidx < 128: + ti = tidx // 32 + ln = tidx % 32 + gg = ln // 4 + tt = ln % 4 + ha = 2 * tt + hb = ha + 1 + x0 = Float32(0.0) + x1 = Float32(0.0) + x2 = Float32(0.0) + x3 = Float32(0.0) + for src in cutlass.range_constexpr(8): + fa = ld_shared_f32(fac + (ha * 9 + src) * 4) + fb = ld_shared_f32(fac + (hb * 9 + src) * 4) + y0, y1, y2, y3 = ld_shared_f32x4(recv_addr + src * block + (ti * 32 + ln) * 16) + x0 = x0 + fa * y0 + x1 = x1 + fb * y1 + x2 = x2 + fa * y2 + x3 = x3 + fb * y3 + inv_a = ld_shared_f32(fac + (ha * 9 + 8) * 4) + inv_b = ld_shared_f32(fac + (hb * 9 + 8) * 4) + ch = 64 * rank + 16 * ti + gg + w0 = bf16_bits(x0 * inv_a) + w1 = bf16_bits(x1 * inv_b) + w2 = bf16_bits(x2 * inv_a) + w3 = bf16_bits(x3 * inv_b) + if const_expr(self.o_rope): + # The last CTA's channels 448 .. 511 are the RoPE ones. + if rank == 7: + pair = 8 * ti + gg // 2 + w0, w1 = self._rope_back_lanes(sRope, pair, gg % 2, w0, w1) + w2, w3 = self._rope_back_lanes(sRope, pair + 4, gg % 2, w2, w3) + out_row = Int64(out.toint()) + Int64(row) * Int64(8 * HEAD_DIM * 2) + st_global_b16(out_row + Int64((ha * HEAD_DIM + ch) * 2), w0) + st_global_b16(out_row + Int64((hb * HEAD_DIM + ch) * 2), w1) + st_global_b16(out_row + Int64((ha * HEAD_DIM + ch + 8) * 2), w2) + st_global_b16(out_row + Int64((hb * HEAD_DIM + ch + 8) * 2), w3) + cluster_wait_aligned() @cute.jit def _channel_owner(self, h, d): @@ -1729,8 +2595,12 @@ def direct_sparse_decode( inputs_ready: bool = False, main_ready: bool = False, query_rope=None, + output_rope: bool = False, slots_out: torch.Tensor | None = None, + plain_edge: bool = False, bulk_exchange: bool = True, + rows_out: torch.Tensor | None = None, + rows_in: torch.Tensor | None = None, ) -> torch.Tensor: """Sparse attention of ``q [T, H, 512]`` (BF16) over its selected packed cache rows. @@ -1739,17 +2609,25 @@ def direct_sparse_decode( logical positions mapped in-kernel when ``main_mapping = (page_table, page_size, max_positions, requests, visible)`` is given (CSA2's owner page-table rules). ``sink [H]`` FP32 logits, ``scale`` the softmax scale. Returns BF16 ``[T, H, 512]`` (``output`` if given). - ``inputs_ready`` / ``main_ready``: the query, SWA slots and page tables / the main - selections and rows were final before the launch preceding this one on the stream started. + ``inputs_ready``: the query, SWA slots and page tables were final before the launch preceding + this one on the stream started. ``main_ready``: the main selections and rows are final when + this launch starts (the launch preceding it releases it only after its own dependency wait). + ``plain_edge``: plain (not programmatic) edges from the producers; dependents still start early. ``query_rope = (positions, cos_sin)``: ``q`` is the projected query before its interleaved RoPE (int32 positions ``[T]``, FP32 table of 32 cos then 32 sin per position), rotated in the - launch. + launch. ``output_rope`` (with ``query_rope``): the output's last 64 channels of every head are + written rotated back by that angle, as ``fp8_grouped_lowm(inverse_rope=...)`` rotates them on + load with the same table. ``slots_out`` (int64 ``[T, Wm]``, with ``main_mapping``): the launch also stores the slot every main selection resolves to (-1 when invalid), so a later launch over the same selections, owner and visible lengths can take them as ``main_indices`` unmapped. ``bulk_exchange``: the partial outputs go to their owners as bulk copies when shared memory allows (same values). + + ``rows_out`` (uint8 ``[T, main_rows_bytes(...)]``, main rows not ``main_ready``): the launch + also stores its decoded main rows; ``rows_in`` (with ``main_ready``): a later launch over the + same main slots of the same pool takes them from there instead of decoding them (same values). """ import cuda.bindings.driver as cuda import cutlass @@ -1837,6 +2715,8 @@ def widest(pool, widths): raise ValueError( "CSA2 direct decode query RoPE needs int32 positions and a 64-wide table" ) + elif output_rope: + raise ValueError("CSA2 direct decode rotates its output back by the query's RoPE angle") q_rope = query_rope is not None swa_index64 = swa_indices.dtype == torch.int64 main_index64 = has_main and main_indices.dtype == torch.int64 @@ -1847,7 +2727,24 @@ def widest(pool, widths): or not slots_out.is_contiguous() ): raise ValueError("CSA2 direct decode slot export needs mapped main rows and int64 [T, Wm]") + image = main_rows_bytes(q_heads, swa_width, main_width, q.device) + for rows_io, ready in ((rows_out, False), (rows_in, True)): + if rows_io is not None and ( + not image + or bool(main_ready) != ready + or rows_io.dtype != torch.uint8 + or rows_io.shape != (rows, image) + or not rows_io.is_contiguous() + or rows_io.data_ptr() % 16 + ): + raise ValueError( + "CSA2 direct decode main-row images need a keys-as-M launch and uint8 " + "[T, main_rows_bytes] (rows_out without main_ready, rows_in with it)" + ) use_pdl = get_env_enable_pdl() + fmul2 = _datacenter_blackwell(q.device.index) + v2 = _v2_engaged(q_heads, swa_width, main_width, q.device.index) + main_prewait = bool(main_ready) and not inputs_ready and _datacenter_blackwell(q.device.index) config = ( heads, swa_width, @@ -1859,12 +2756,20 @@ def widest(pool, widths): bool(inputs_ready), bool(main_ready), use_pdl, + bool(plain_edge) and use_pdl, swa_vec, main_vec, q_heads, q_rope, bool(bulk_exchange), slots_out is not None, + fmul2, + v2, + main_prewait, + bool(output_rope), + _datacenter_blackwell(q.device.index), + rows_out is not None, + rows_in is not None, ) key = (q.device.index, *config) swa_index_type = cutlass.Int64 if swa_index64 else cutlass.Int32 @@ -1902,6 +2807,7 @@ def widest(pool, widths): _pointer(q_positions, cutlass.Int32, 4), _pointer(q_table, cutlass.Float32, 4), _pointer(slots_out, cutlass.Int64, 8), + _pointer(rows_out if rows_out is not None else rows_in, cutlass.Uint8, 16), cuda.CUstream(torch.cuda.current_stream(q.device).cuda_stream), ) compiled = _COMPILED.get(key) @@ -1909,7 +2815,10 @@ def widest(pool, widths): if torch.cuda.is_current_stream_capturing(): raise RuntimeError("Warm up CSA2 direct decode before CUDA Graph capture") kernel = _kernel_type()(*config) - compiled = _COMPILED[key] = cute.compile(kernel, *args) + compiled = cute.compile(kernel, *args) + if v2: + _set_smem_carveout(compiled, _ROOM_CARVEOUT) + _COMPILED[key] = compiled compiled(*args) return output @@ -1918,4 +2827,5 @@ def widest(pool, widths): "MAX_ROWS", "direct_decode_device_supported", "direct_sparse_decode", + "main_rows_bytes", ] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py index 00cb70213e23..34b466c6ec0e 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py @@ -4078,11 +4078,12 @@ class QuantizeScatterKernel: quantization groups reduce with butterfly shuffles. """ - def __init__(self, jobs, pdl: bool = False): + def __init__(self, jobs, pdl: bool = False, release_after_wait: bool = False): if not 1 <= len(jobs) <= _MAX_QUANTIZE_JOBS: raise ValueError("A CSA2 quantize launch takes one to four jobs") self.jobs = tuple(jobs) self.pdl = pdl + self.release_after_wait = release_after_wait self.threads = max(job.threads for job in self.jobs) # A norm row wider than a warp reduces through ``partial`` and a CTA barrier, so it # cannot share its CTA with other rows. @@ -4280,7 +4281,7 @@ def kernel( ): cta, _, _ = cute.arch.block_idx() tid, _, _ = cute.arch.thread_idx() - pdl_prologue(self.pdl, False) + pdl_prologue(self.pdl, self.release_after_wait) smem = SmemAllocator() partial = smem.allocate_tensor( Float32, cute.make_layout(max(self.threads // 32, 1)), 16 @@ -4662,8 +4663,8 @@ class QuantizeScatterVecKernel(QuantizeScatterKernel): THREADS = 256 - def __init__(self, jobs, pdl: bool = False): - super().__init__(jobs, pdl) + def __init__(self, jobs, pdl: bool = False, release_after_wait: bool = False): + super().__init__(jobs, pdl, release_after_wait) if any(job.head_dim not in (128, 512) for job in self.jobs): raise ValueError("QuantizeScatterVecKernel takes 128- or 512-channel rows") self.threads = self.THREADS @@ -7619,8 +7620,9 @@ def split_job(values: torch.Tensor, data: torch.Tensor, scales: torch.Tensor) -> ) -def quantize_scatter_jobs(jobs) -> None: - """Run up to four quantize jobs (``None`` entries skipped) as one launch.""" +def quantize_scatter_jobs(jobs, *, release_after_wait: bool = False) -> None: + """Run up to four quantize jobs (``None`` entries skipped) as one launch; with + ``release_after_wait`` (under PDL) the next kernel may launch right after its wait.""" jobs = [job for job in jobs if job is not None] if not jobs: return @@ -7643,14 +7645,15 @@ def quantize_scatter_jobs(jobs) -> None: job.eps, ] pdl = _pdl() + release = pdl and release_after_wait vec = ( all(job.vec_ok for job in jobs) and sum(job.rows * job.spec.head_dim for job in jobs) >= _QUANTIZE_VEC_MIN_ELEMENTS ) kernel = ns.QuantizeScatterVecKernel if vec else ns.QuantizeScatterKernel _launch( - ("quantize", vec, tuple(job.spec for job in jobs), pdl, device.index), - lambda: kernel([job.spec for job in jobs], pdl), + ("quantize", vec, tuple(job.spec for job in jobs), pdl, release, device.index), + lambda: kernel([job.spec for job in jobs], pdl, release), *args, _stream(device), ) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py index 04919ec6e787..db6c95d0fb9d 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/metadata.py @@ -88,6 +88,10 @@ def csa2_side_stream(device, index: int = 0) -> torch.cuda.ExternalStream: return private_stream(torch.device(device).index, f"csa2_side_{index}") +# ``csa2_side_stream`` index of a captured decode forward's SWA slot refresh. +_SWA_REFRESH_STREAM = 3 + + @functools.cache def _layout_layers(layout) -> tuple: """``layout.layer`` of every layer of the (immutable) ``layout``.""" @@ -245,6 +249,10 @@ class CSA2TrtllmMetadata(TrtllmAttentionMetadata): _csa2_pending_prepare: tuple | None = None _csa2_model_driven: bool = False _csa2_graph_reserved: bool = False + # (done event, model stream) of this forward's SWA slot refresh on its side stream. + _csa2_swa_fork: tuple | None = None + # A decode forward's SWA slot refresh launches at its first CSA2 layer's start. + _csa2_swa_fork_armed: bool = False # Set by ``_allocate``: (the allocation, the extra pool's first 512-element row) when both staging # pools are one bank for the ``gen_fmha`` core. gen_fmha_bank: tuple[torch.Tensor, int] | None = None @@ -1780,6 +1788,17 @@ def _ensure_swa_slots(self) -> None: """ if self._csa2_pending_prepare is not None: self._prepare_deferred() + fork = self._csa2_swa_fork + if fork is not None: + # Every reader's stream waits for the forked refresh; the model stream's wait orders all + # later readers, so it ends the fork (outside a capture, a fork of an aborted capture). + if torch.cuda.is_current_stream_capturing(): + current = torch.cuda.current_stream() + current.wait_event(fork[0]) + if current == fork[1]: + self._csa2_swa_fork = None + else: + self._csa2_swa_fork = None # None: not prepared by prepare_csa2. if getattr(self, "_csa2_swa_resolved", None) is not False: return @@ -2690,6 +2709,58 @@ def begin_model_forward(self) -> None: self.csa2_precomputed_kv_layers = set() self.csa2_replay_query_rows = None self.csa2_decoder_capture_lens = None + self._csa2_swa_fork = None + self._csa2_swa_fork_armed = self.num_contexts == 0 + + def _launch_swa_refresh_fork(self) -> None: + """Launch a captured attention-TP decode forward's SWA slot refresh on a side stream after + the model stream's work so far; the model stream's first ``_ensure_swa_slots`` joins it.""" + from tensorrt_llm._torch.pyexecutor.breakable_cuda_graph import is_in_breakable_cuda_graph + from tensorrt_llm._utils import is_blackwell_datacenter + + from . import kernel + + positions = getattr(self, "csa2_positions", None) + if ( + positions is None + or not positions.is_cuda + or (self.mapping is not None and self.mapping.enable_attention_dp) + or not is_blackwell_datacenter() + ): + return + side = csa2_side_stream(positions.device, _SWA_REFRESH_STREAM) + if ( + not torch.cuda.is_current_stream_capturing() + or is_in_breakable_cuda_graph() + or getattr(self, "_csa2_swa_resolved", None) is not False + or self._csa2_pending_prepare is not None + or getattr(self, "_csa2_swa_descriptors", None) is None + or positions.numel() == 0 + or not kernel.dsl_available() + ): + return + manager = self.kv_cache_manager + layers, count = max(self._csa2_swa_page_tables, default=-1) + 1, positions.numel() + # The refresh's output slabs come from the model stream's allocations. + self._get_csa2_buffer("swa_reads", (layers, count, manager.layout.window_size), torch.int64) + self._get_csa2_buffer("swa_writes", (layers, count), torch.int64) + self._get_csa2_buffer("visible", (len(manager.layout.compress_ratios), count), torch.int64) + caller = torch.cuda.current_stream() + side.wait_stream(caller) + with torch.cuda.stream(side): + self._ensure_swa_slots() + done = torch.cuda.Event() + done.record() + self._csa2_swa_fork = (done, caller) + + def join_swa_refresh_fork(self) -> None: + """End of a model forward: the current stream joins a forked SWA slot refresh no reader + consumed, so a capture never ends with the side stream unjoined.""" + fork = self._csa2_swa_fork + if fork is not None: + if torch.cuda.is_current_stream_capturing(): + torch.cuda.current_stream().wait_event(fork[0]) + self._csa2_swa_fork = None def on_update_kv_lens(self) -> None: """Refresh query/cache addresses after device-side acceptance correction. @@ -2862,8 +2933,9 @@ def _refresh_owner_slots( def reset_routing(self) -> None: """Begin one packed forward; graph replay recomputes captured producers.""" self.csa2_indices = {} - # Main slots an index-source layer's direct decode exported this forward. + # Main slots and decoded main rows an index source's direct decode exported this forward. self.csa2_direct_slots = {} + self.csa2_direct_rows = {} self.csa2_candidates = {} self.csa2_candidate_blocks = {} self.csa2_candidate_counts = {} @@ -3628,6 +3700,10 @@ def _prepare_deferred(self) -> None: self._refresh_live_endpoints() def get_compression_batch(self, owner: int): + if self._csa2_swa_fork_armed: + # The refresh forks here to overlap the first CSA2 layer's projections. + self._csa2_swa_fork_armed = False + self._launch_swa_refresh_fork() if self._csa2_pending_prepare is not None: self._prepare_deferred() if ( diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/module.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/module.py index 394fbbc107aa..1486deb6d5fa 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/module.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/module.py @@ -27,7 +27,7 @@ from tensorrt_llm._torch.modules.multi_stream_utils import do_multi_stream from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.utils import MxFp8QuantizedTensor, is_torch_compiling -from tensorrt_llm._utils import get_sm_version, is_sm_100f +from tensorrt_llm._utils import get_sm_version, is_blackwell_datacenter, is_sm_100f from tensorrt_llm.functional import PositionEmbeddingType, RotaryScalingType from tensorrt_llm.mapping import Mapping from tensorrt_llm.math_utils import pad_up @@ -49,10 +49,18 @@ # GB300, cold L2: the dense CuTe DSL GEMM beats every native config of O-B (5120 x 8192) from # 384 rows; below, a native config is faster. _DENSE_O_B_MIN_ROWS = 384 +# Fused decode steps of up to 64 rows prefetch the O-A / O-B weights into L2 on a side stream, from +# one-warp CTAs on the SMs the decode leaves free (SM100/SM103). +_PREFETCH_OUTPUT_MAX_ROWS = 64 +_PREFETCH_OUTPUT_CTAS = 8 +# 24 us: a CTA stops issuing then, so a prefetch that gets its SMs late never outlasts O-B. +_PREFETCH_OUTPUT_BUDGET_NS = 24000 +# The fused decode splits a step's tokens over at most this many CTAs each (kMaxCombineSplits). +_DECODE_MAX_SPLITS = 8 # 8-column tiles per CTA of the small-M Q-A + KV and Q-B projections of full 16-row steps: half the -# CTAs, each loading the step's rows once for both (the same bits). Smaller steps keep one tile per -# CTA, whose extra CTAs keep more weight loads in flight. +# CTAs, each loading the step's rows once for both (the same bits). Smaller steps take +# ``lowm_tiles``; one tile per CTA above 8 rows keeps more weight loads in flight. _LOWM_QUERY_TILES = 2 _LOWM_TWO_TILE_MIN_ROWS = 16 @@ -60,7 +68,9 @@ def _lowm_tiles(n: int, rows: int) -> int: if rows >= _LOWM_TWO_TILE_MIN_ROWS and n % (8 * _LOWM_QUERY_TILES) == 0: return _LOWM_QUERY_TILES - return 1 + from tensorrt_llm._torch.modules.low_m_gemm import lowm_tiles + + return lowm_tiles(n, rows) # Per (device, stream): the fused context kernel's O-A input buffers, grown to the largest forward. @@ -191,7 +201,10 @@ def __init__( self._descriptors_done = torch.cuda.Event() if aux_stream is not None else None self._query_done = torch.cuda.Event() if aux_stream is not None else None self._touch_done = torch.cuda.Event() if aux_stream is not None else None + self._qkv_done = torch.cuda.Event() if aux_stream is not None else None + self._rows_done = torch.cuda.Event() if aux_stream is not None else None self._touch_sink = None + self._prefetch_events = None self._overlap_query_kv = True self._fuse_index_q = sparse_params.fuse_index_q if sparse_params.fuse_packed_output_rope and pos_embd_params.is_neox: @@ -523,7 +536,11 @@ def _project_query_and_swa( late_stream: torch.cuda.Stream | None = None, fp8_query: _FP8Query | None = None, prequantized: MxFp8QuantizedTensor | None = None, + swa_ready: torch.cuda.Event | None = None, + smem_carveout: int = 0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + """``(qr, q, swa)``; ``swa_ready`` is recorded right after a small decode step's Q-A + KV + projection (``_lowm_projects_qkv``); ``smem_carveout``: of the small-M GEMMs.""" if not decode_only and self._can_fuse_prefill_projection(hidden_states): return self._project_prefill_query_and_swa(hidden_states) if ( @@ -542,9 +559,17 @@ def _project_query_and_swa( split_n=self.wq_a.weight.shape[0], scale_swizzled=True, n_tiles=_lowm_tiles(self._fused_qkv_weight.shape[0], hidden_states.shape[0]), + smem_carveout=smem_carveout, ) + if swa_ready is not None: + swa_ready.record() return self._project_decode_query( - latent, swa, query_stream, late_stream=late_stream, fp8_query=fp8_query + latent, + swa, + query_stream, + late_stream=late_stream, + fp8_query=fp8_query, + smem_carveout=smem_carveout, ) fused = decode_only and hidden_states.shape[0] > 0 and self.takes_mxfp8_input if not fused: @@ -597,6 +622,7 @@ def _project_query_and_swa( query_stream, late_stream=late_stream, fp8_query=fp8_query, + smem_carveout=smem_carveout, ) def _project_decode_query( @@ -607,6 +633,7 @@ def _project_decode_query( *, late_stream: torch.cuda.Stream | None, fp8_query: _FP8Query | None, + smem_carveout: int = 0, ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]: """``(qr, q, swa)`` of a decode step from its Q-A latent, on ``query_stream`` if given.""" from .query_projection import fp8_query_single_tile_row @@ -631,6 +658,7 @@ def _project_decode_query( tune=_FLASHINFER_MXFP8_AUTOTUNE_ACTIVE.get(), late_stream=late_stream, fp8_query=fp8_query, + smem_carveout=smem_carveout, ) return qr, q, swa @@ -763,6 +791,7 @@ def _project_query_latent( tune: bool, late_stream: torch.cuda.Stream | None = None, fp8_query: _FP8Query | None = None, + smem_carveout: int = 0, ) -> tuple[torch.Tensor, torch.Tensor | None]: """Return ``(qr, q)``; ``q`` is None once ``fp8_query`` received the projection.""" rank = latent.shape[1] @@ -838,6 +867,7 @@ def _project_query_latent( (self.q_norm.weight, self.q_norm.variance_epsilon) if norm_in_q_b else None ), n_tiles=_lowm_tiles(self.wq_b.weight.shape[0], latent.shape[0]), + smem_carveout=smem_carveout, ) elif fused: q = mxfp8_query_gemm( @@ -954,6 +984,39 @@ def _touch_output_weights(self, rows: int) -> bool: self._touch_done.record() return True + def _output_prefetch_ctas(self, rows: int) -> int: + """CTAs of the O-A/O-B weight prefetch beside a fused decode of ``rows`` rows (0: none).""" + weights = (self.o_a_proj, self.o_b_proj.weight) + if not ( + 0 < rows <= _PREFETCH_OUTPUT_MAX_ROWS + and is_blackwell_datacenter() + # The prefetch moves 128x128-byte boxes. + and all( + w.dtype == torch.float8_e4m3fn + and w.is_contiguous() + and w.shape[-1] % 128 == 0 + and w.numel() // w.shape[-1] % 128 == 0 + for w in weights + ) + ): + return 0 + sms = torch.cuda.get_device_properties(weights[0].device).multi_processor_count + # The decode CTAs need whole SMs and take at most this many. + return max(0, min(_PREFETCH_OUTPUT_CTAS, sms - min(_DECODE_MAX_SPLITS, sms // rows) * rows)) + + def _prefetch_output_weights(self, ctas: int) -> None: + """Prefetch the O-A and O-B weights into L2 on a side stream from the fused decode's launch + (``_prefetch_events[0]``, recorded by the backend) on; O-B joins ``_prefetch_events[1]``.""" + from .projection import prefetch_output_weights + + decode_start, done = self._prefetch_events + with torch.cuda.stream(csa2_side_stream(self.o_a_proj.device, 2)): + decode_start.wait() + prefetch_output_weights( + self.o_a_proj, self.o_b_proj.weight, ctas, _PREFETCH_OUTPUT_BUDGET_NS + ) + done.record() + def _q_amax_word(self, device: torch.device) -> torch.Tensor | None: """The int32 word Q's producer raises to Q's amax bits; staging consumes and clears it.""" if self._fp8_q_word is None: @@ -1089,6 +1152,34 @@ def _prepare_auxiliary( index_weights = self._project_index_weights(hidden_states) return index_weights, main_kv, index_k + def _publish_rows(self, attn_metadata, swa, positions, compressed_positions, auxiliary) -> None: + """Publish the entered layer's SWA (and a Full layer's global) rows on a side stream after + Q-A + KV (``_qkv_done``) and the compressor (``_prepare_done``); ``_rows_done`` follows.""" + _, main_kv, index_k = auxiliary + stream = csa2_side_stream(swa.device, 4) + with torch.cuda.stream(stream): + self._qkv_done.wait() + if self.layer.mode == CSA2Mode.FULL: + self._prepare_done.wait() + for tensor in (swa, positions, compressed_positions, main_kv, index_k): + if tensor is not None: + tensor.record_stream(stream) + main_transform, index_transform = self.global_transforms(compressed_positions) + self.backend.publish_layer_rows( + CSA2ForwardState( + metadata=attn_metadata, + swa_kv=swa, + main_kv=main_kv, + index_k=index_k, + swa_transform=self._row_transform( + positions, norm=(self.kv_norm.weight, self.kv_norm.variance_epsilon) + ), + main_transform=main_transform, + index_transform=index_transform, + ) + ) + self._rows_done.record() + def _project_index_weights(self, hidden_states): projected = bf16_projection(self.index_weights_proj, hidden_states) # One launch for ``(projected * scale).to(dtype)``, bit for bit. @@ -1100,6 +1191,21 @@ def _project_index_weights(self, hidden_states): ), ) + def _packs_index_q(self, rows: int, positions: torch.Tensor) -> bool: + """Whether the index-query projection of a decode-graph step of ``rows`` rows packs the + queries as the cache write would (``mxfp8_lowm_index_query``), off the index chain.""" + return ( + self.layer.mode in (CSA2Mode.FULL, CSA2Mode.REINDEX) + and not self._fuse_index_q + and not self.mapping.enable_attention_dp + and is_blackwell_datacenter() + and not self.rotary_emb.is_neox + and self.index_head_dim == 128 + and self.qk_rope_head_dim == 64 + and positions.dtype == torch.int32 + and self._lowm_projects(self.index_wq_b, rows) + ) + def _project_index_q(self, qr, positions): """Index Q of BF16 ``qr``, or of its MXFP8 ``(values, scales)`` pair.""" if self.layer.mode not in (CSA2Mode.FULL, CSA2Mode.REINDEX): @@ -1127,6 +1233,16 @@ def _project_index_q(self, qr, positions): self._fp8_alpha, ) return packed.view(-1, self.index_heads, 64), scale.view(-1, self.index_heads) + if self._packs_index_q(qr.shape[0], positions): + from tensorrt_llm._torch.modules.low_m_gemm import mxfp8_lowm_index_query + + return mxfp8_lowm_index_query( + qr.contiguous(), + self.index_wq_b.weight, + self.index_wq_b.weight_scale, + positions, + self.rotary_emb.rotary_cos_sin.view(-1, 2, self.qk_rope_head_dim // 2), + ) if ( self._lowm_projects(self.index_wq_b, qr.shape[0]) and not self.rotary_emb.is_neox @@ -1169,12 +1285,12 @@ def _lowm_o_a(self, rows: int) -> bool: and (depth // default_k_warps(depth)) % self.qk_head_dim == 0 ) - def _project_output_small_m(self, output, positions): - """O-A with inverse RoPE fused, then O-B, for a small step; None when not applicable.""" + def _small_m_o_a_fits(self, output, positions) -> bool: + """Whether ``_project_output_small_m`` projects ``output``.""" rows = output.shape[0] positions = positions.reshape(-1) groups, rank, depth = self.o_a_proj.shape - if not ( + return ( self._small_m(rows) and self.o_a_proj.dtype == torch.float8_e4m3fn and self.o_a_proj_scale is not None @@ -1187,7 +1303,40 @@ def _project_output_small_m(self, output, positions): and positions.dtype == torch.int32 and positions.numel() == rows and positions.is_contiguous() - ): + ) + + def _rope_tables_match(self) -> bool: + """Whether the inverse RoPE's table is the forward one (one shared tensor), whose + coefficients the direct decode holds.""" + table, inverse = self.rotary_emb.rotary_cos_sin, self.inverse_rotary_emb.rotary_cos_sin + return (table.data_ptr(), table.shape, table.stride()) == ( + inverse.data_ptr(), + inverse.shape, + inverse.stride(), + ) + + def _o_a_takes_rotated_back(self, output, positions) -> bool: + """Whether the direct decode writes ``output`` rotated back for an O-A on the small-M + warp-MMA kernel, which then skips its inverse RoPE on load (SM100/SM103).""" + return ( + is_blackwell_datacenter() + and self.projection_quantization != "bf16" + and self._o_a_fp8_block128 + and output.dtype == torch.bfloat16 + and self._small_m_o_a_fits(output, positions) + and self._lowm_o_a(output.shape[0]) + and self._rope_tables_match() + ) + + def _project_output_small_m(self, output, positions, rotated_back: bool = False): + """O-A with inverse RoPE fused (unless the output is ``rotated_back`` already), then O-B, + for a small step; None when not applicable.""" + rows = output.shape[0] + positions = positions.reshape(-1) + groups, rank, depth = self.o_a_proj.shape + if not self._small_m_o_a_fits(output, positions): + if rotated_back: + raise ValueError("CSA2 rotated-back attention output needs the small-M O-A") return None from .projection import ( inverse_rope_block128_small_m, @@ -1207,26 +1356,52 @@ def _project_output_small_m(self, output, positions): and not (linear.reduce_output and linear.use_fused_gemm_allreduce) ) if output.dtype == torch.bfloat16 and self._lowm_o_a(rows): - from tensorrt_llm._torch.modules.low_m_gemm import fp8_grouped_lowm, mxfp8_lowm_linear + from tensorrt_llm._torch.modules.low_m_gemm import ( + fp8_grouped_lowm, + lowm_tiles, + mxfp8_lowm_linear, + ) - # Both kernels load their weights before the PDL wait: O-B's stream in under O-A. + projects = quantize and self._lowm_projects(linear, rows) + # A boundary that takes O-B is released after the decode: its weights come from L2 (its + # entry boundary prefetched them), and its loads beside the decode would slow it. + defers = bool( + projects + and getattr(self, "defer_ob_to_boundary", False) + and not linear.reduce_output + ) + # Otherwise both kernels load their weights before the PDL wait: O-B's stream in under O-A. latent = fp8_grouped_lowm( output.reshape(rows, -1).contiguous(), self.o_a_proj.view(groups * rank, depth), self.o_a_proj_scale.view(groups * rank // 128, depth // 128), rank, - inverse_rope=( + inverse_rope=None + if rotated_back + else ( positions, self.inverse_rotary_emb.rotary_cos_sin.view(-1, 2, self.qk_rope_head_dim // 2), self.qk_head_dim, ), + n_tiles=lowm_tiles(groups * rank, rows, rank), + release_after_wait=defers, ) - if not (quantize and self._lowm_projects(linear, rows)): + if not projects: return linear(latent) + if defers: + from tensorrt_llm._torch.modules.low_m_gemm import mxfp8_linear_scale + from tensorrt_llm._torch.modules.mhc.mhc_lowm import defer_ob_projection + + # The all-reduce boundary that follows runs this projection in its own launch. + deferred = defer_ob_projection(latent, linear.weight, mxfp8_linear_scale(linear)) + if deferred is not None: + return deferred projected = mxfp8_lowm_linear( latent, linear.weight, linear.weight_scale, scale_swizzled=True ) return linear.all_reduce(projected) if linear.reduce_output else projected + if rotated_back: + raise ValueError("CSA2 rotated-back attention output needs the small-M O-A") o_lora = inverse_rope_block128_small_m( output.reshape(rows, -1), positions, @@ -1336,7 +1511,14 @@ def _project_prequantized_output(self, data, scale, out=None): return projected def _project_output( - self, output, positions, *, inverse_rope_applied: bool = False, prequantized=None, out=None + self, + output, + positions, + *, + inverse_rope_applied: bool = False, + prequantized=None, + out=None, + rotated_back: bool = False, ): if prequantized is not None: projected = self._project_prequantized_output(*prequantized, out=out) @@ -1345,6 +1527,8 @@ def _project_output( return project_sparse_attn_output( self, [output.flatten(1)], positions, prequantized=prequantized ) + if rotated_back: + return self._project_output_small_m(output, positions, rotated_back=True) if self._o_a_fp8_block128 and inverse_rope_applied: raise ValueError("Grouped 128-block FP8 owns its fused inverse-RoPE/quantization") if self.projection_quantization == "bf16": @@ -1668,7 +1852,25 @@ def forward( hidden_states.shape[0], 0, pad_up(extra_width, 128), hidden_states.device ) ) + # The query projections overlap the decode they release; at its split they can share SMs. + projection_carveout = ( + self.backend.direct_decode_carveout(pad_up(extra_width, 128), hidden_states.device) + if direct and query_overlap + else 0 + ) gen_fmha = self._gen_fmha_decode(hidden_states.shape[0], attn_metadata) + # A small direct-decode step whose index-query projection packs the queries publishes the + # layer's cache rows off the index chain (``_publish_rows``). + publish_rows = ( + late_query + and direct + and hidden_states.dtype == torch.bfloat16 + and self._lowm_projects_qkv(hidden_states.shape[0]) + and self._packs_index_q(hidden_states.shape[0], positions) + ) + if publish_rows: + # On this stream: a slot resolution it may launch precedes the side stream's fork. + attn_metadata.enter_layer(self.layer) quant_q = q_amax = None unit_q = touched = False if fused_query: @@ -1716,7 +1918,11 @@ def forward( late_stream=self.aux_stream if late_query else None, fp8_query=fp8_query, prequantized=hidden_states_mxfp8, + swa_ready=self._qkv_done if publish_rows else None, + smem_carveout=projection_carveout, ) + if publish_rows: + self._publish_rows(attn_metadata, swa, positions, compressed_positions, auxiliary) if fp8_query is not None and fp8_query.applied: # Staging reads only the E4M3 Q; this BF16 Q keeps the shapes and is never read. quant_q = fp8_query.quant_q @@ -1738,6 +1944,8 @@ def forward( self._prepare_done.record() touched = self._touch_output_weights(q.shape[0]) elif late_query: + if publish_rows: + self._rows_done.wait() self._query_done.record() if query_overlap or late_query: q.record_stream(torch.cuda.current_stream(hidden_states.device)) @@ -1756,6 +1964,9 @@ def forward( # The cache write needs the compressor's rows (and the index weights unless they # were split onto the second side stream, which the indexer joins: ``index_ready``). self._prepare_done.wait() + if publish_rows and self.layer.mode == CSA2Mode.FULL: + # The indexer scores this step's index rows. + self._rows_done.wait() current = torch.cuda.current_stream(hidden_states.device) for tensor in auxiliary: if tensor is not None: @@ -1825,6 +2036,14 @@ def forward( positions.reshape(-1), self.inverse_rotary_emb.rotary_cos_sin, ) + prefetch_ctas = ( + self._output_prefetch_ctas(q.shape[0]) + if output_mxfp8 is not None and not gen_fmha and self.backend.fused_decode + else 0 + ) + if prefetch_ctas and self._prefetch_events is None: + self._prefetch_events = (torch.cuda.Event(), torch.cuda.Event()) + rotated_back = direct and self._o_a_takes_rotated_back(q, positions) state = CSA2ForwardState( metadata=attn_metadata, swa_kv=swa, @@ -1844,6 +2063,9 @@ def forward( output_fp8=output_fp8, output_mxfp8=output_mxfp8, query_rope=(positions.reshape(-1), self.rotary_emb.rotary_cos_sin) if direct else None, + output_rope=rotated_back, + decode_start=self._prefetch_events[0] if prefetch_ctas else None, + rows_published=publish_rows, ) if unit_q and output_fp8 is not None: # The fused context kernel writes the O-A input instead of the BF16 output. @@ -1917,6 +2139,8 @@ def forward( compute(tile.flatten(1), metadata, args) else: self.backend.forward(tile.flatten(1), None, None, metadata, forward_args=args) + if prefetch_ctas: + self._prefetch_output_weights(prefetch_ctas) if state.output_fp8_written: projected = self._project_output( output, positions, prequantized=output_fp8[:2], out=out @@ -1931,10 +2155,15 @@ def forward( if output.stride(0) == 0: raise RuntimeError("CSA2 fused context kernel did not write the O-A input") projected = self._project_output( - output, positions, inverse_rope_applied=inverse_rope_applied + output, + positions, + inverse_rope_applied=inverse_rope_applied, + rotated_back=rotated_back, ) if touched: torch.cuda.current_stream(hidden_states.device).wait_event(self._touch_done) + if prefetch_ctas: + torch.cuda.current_stream(hidden_states.device).wait_event(self._prefetch_events[1]) if out is None or projected is out: return projected return out.copy_(projected) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/params.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/params.py index 24820464ddee..b9f1294ef813 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/params.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/params.py @@ -285,12 +285,18 @@ class CSA2ForwardState: output_mxfp8: tuple[torch.Tensor, ...] | None = None # (positions, cos/sin) when the query still awaits its RoPE, which the direct decode applies. query_rope: tuple[torch.Tensor, torch.Tensor] | None = None + # With ``query_rope``: the direct decode writes its output rotated back for the O-A. + output_rope: bool = False swa_transform: RowTransform | None = None main_transform: RowTransform | None = None index_transform: RowTransform | None = None # Recorded on a side stream after the index weights and the paged indexer descriptors; # the indexer waits for it before scoring. index_ready: torch.cuda.Event | None = None + # Recorded on the model stream right before the fused decode kernel launches. + decode_start: torch.cuda.Event | None = None + # The module entered the layer and published its cache rows (``publish_layer_rows``) itself. + rows_published: bool = False @dataclass(frozen=True) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/projection.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/projection.py index d4c10b57a561..c930f8ed519b 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/projection.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/projection.py @@ -133,6 +133,63 @@ def l2_touch(tensors: tuple[torch.Tensor, ...], sink: torch.Tensor, block: int = _l2_touch_kernel[(sink.numel(),)](*args, sink, block, num_warps=8) +_PREFETCH_COMPILED = {} +# Per device: a word whose only use is to carry the TVM FFI stream into the prefetch launch. +_PREFETCH_ANCHOR: dict[int, torch.Tensor] = {} + + +def prefetch_output_weights( + o_a_weight: torch.Tensor, + o_b_weight: torch.Tensor, + ctas: int, + budget_ns: int, +) -> bool: + """TMA-prefetch E4M3 O-A ``[groups, R, D]`` and O-B ``[hidden, groups*R]`` into L2 from + ``ctas`` CTAs; False if a capture would compile the kernel.""" + import cutlass + import cutlass.cute as cute + + from tensorrt_llm._torch.kernels.blackwell.csa2_weight_prefetch import Csa2WeightPrefetch + from tensorrt_llm._torch.kernels.blackwell.utils import make_ptr + + shapes = tuple( + (weight.numel() // weight.shape[-1], weight.shape[-1]) + for weight in (o_a_weight, o_b_weight) + ) + device = o_a_weight.device + key = (device.index, shapes) + compiled = _PREFETCH_COMPILED.get(key) + anchor = _PREFETCH_ANCHOR.get(device.index) + if compiled is None or anchor is None: + if torch.cuda.is_current_stream_capturing(): + return False + if anchor is None: + anchor = _PREFETCH_ANCHOR[device.index] = torch.zeros( + 1, device=device, dtype=torch.int32 + ) + if compiled is None: + pointers = ( + make_ptr( + cutlass.Float8E4M3FN, + weight.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=16, + ) + for weight in (o_a_weight, o_b_weight) + ) + compiled = _PREFETCH_COMPILED[key] = cute.compile( + Csa2WeightPrefetch(*shapes).run, + *pointers, + cute.runtime.from_dlpack(anchor), + ctas, + budget_ns, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--opt-level 2 --enable-tvm-ffi", + ) + compiled(o_a_weight.data_ptr(), o_b_weight.data_ptr(), anchor, ctas, budget_ns) + return True + + # QUANTIZE is a runtime value, so both modes run one compiled kernel and write the same BF16 # rows; it only masks the MXFP8 stores. @triton.jit(do_not_specialize=["M", "QUANTIZE"]) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/quantization.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/quantization.py index 3a7871ad7a6f..bcd9b4a0ba07 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/quantization.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/quantization.py @@ -292,6 +292,7 @@ def store_layer_rows( main=None, index=None, index_q: torch.Tensor | None = None, + release_after_wait: bool = False, ): """Publish one layer's cache rows with a single quantize launch when possible. @@ -331,7 +332,7 @@ def store_layer_rows( if views is not None: jobs.append(split_job(*views)) with torch.cuda.device(swa_values.device): - quantize_scatter_jobs(jobs) + quantize_scatter_jobs(jobs, release_after_wait=release_after_wait) return packed diff --git a/tensorrt_llm/_torch/attention/backends/sparse/csa2/query_projection.py b/tensorrt_llm/_torch/attention/backends/sparse/csa2/query_projection.py index 2868e951a90f..312982a639af 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/csa2/query_projection.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/csa2/query_projection.py @@ -8,6 +8,8 @@ import triton import triton.language as tl +from tensorrt_llm._utils import is_blackwell_datacenter + @cache def _kernel_type() -> type: @@ -321,12 +323,16 @@ def _max_active_clusters(cluster_size: int) -> int: # Rows from which 2-CTA 256x256 tiles beat 256x192 ones on GB300. _FP8_QUERY_WIDE_TILE_MIN_ROWS = 4096 # 4K +# Up to this many rows one 128x256 tile per CTA beats two 128x128 tiles on SM100/SM103. +_FP8_QUERY_WIDE_N_MAX_ROWS = 64 -def _fp8_query_tile(m: int) -> tuple[int, int]: +def _fp8_query_tile(m: int, n: int = 0) -> tuple[int, int]: # Measured on GB300: with only the E4M3 output, 2-CTA 256x192 tiles beat 256x256 ones # from 129 to 3072 rows; from 257 to 384 rows, where a second 256-row tile is mostly # padding, 128x128 tiles beat both. + if m <= _FP8_QUERY_WIDE_N_MAX_ROWS and n % 256 == 0 and is_blackwell_datacenter(): + return (128, 256) if m <= 128 or 256 < m <= 384: return (128, 128) return (256, 192) if m < _FP8_QUERY_WIDE_TILE_MIN_ROWS else (256, 256) @@ -394,7 +400,7 @@ def fp8_query_gemm( ) ): raise ValueError("CSA2 FP8 query needs a contiguous E4M3 [M, N] output and an int32 word") - tile = tile or _fp8_query_tile(m) + tile = tile or _fp8_query_tile(m, n) cluster = cluster or (tile[0] // 128, 1) active = _max_active_clusters(cluster[0] * cluster[1]) if cluster == (tile[0] // 128, 1) and m <= tile[0]: diff --git a/tensorrt_llm/_torch/kernels/blackwell/csa2_weight_prefetch.py b/tensorrt_llm/_torch/kernels/blackwell/csa2_weight_prefetch.py new file mode 100644 index 000000000000..d4bbc82a0445 --- /dev/null +++ b/tensorrt_llm/_torch/kernels/blackwell/csa2_weight_prefetch.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""TMA prefetch of the CSA2 output-projection weights into L2.""" + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync +from cutlass.cutlass_dsl import Int64, T, dsl_user_op + + +@dsl_user_op +def _globaltimer(*, loc=None, ip=None) -> Int64: + return Int64( + llvm.inline_asm( + T.i64(), + [], + "mov.u64 $0, %globaltimer;", + "=l", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +class Csa2WeightPrefetch: + """Prefetch two E4M3 weights ``[rows, k]`` into L2 in 128x128-byte TMA boxes, in the order the + output GEMMs read them; a CTA stops issuing once its time budget is spent.""" + + def __init__(self, first: tuple[int, int], second: tuple[int, int]): + if any(size % 128 for size in (*first, *second)): + raise ValueError("CSA2 weight prefetch needs 128-aligned weights") + self.shapes = (first, second) + + @cute.jit + def run( + self, + first: cute.Pointer, + second: cute.Pointer, + anchor: cute.Tensor, + ctas: cutlass.Int32, + budget_ns: cutlass.Int64, + stream: cuda.CUstream, + ): + """``anchor`` only carries the TVM FFI stream.""" + box = cute.make_layout((128, 128), stride=(128, 1)) + atoms = [] + for pointer, (rows, k) in zip((first, second), self.shapes): + tensor = cute.make_tensor(pointer, cute.make_layout((rows, k), stride=(k, 1))) + atoms.append( + cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(), tensor, box, (128, 128) + ) + ) + + @cute.struct + class SharedStorage: + box: cute.struct.Align[cute.struct.MemRange[first.dtype, 128 * 128], 1024] + + self.shared_storage = SharedStorage + self.kernel(*atoms[0], *atoms[1], ctas, budget_ns).launch( + grid=[ctas, 1, 1], + block=[32, 1, 1], + smem=SharedStorage.size_in_bytes(), + stream=stream, + ) + + @cute.kernel + def kernel( + self, + atom_a: cute.CopyAtom, + mA: cute.Tensor, + atom_b: cute.CopyAtom, + mB: cute.Tensor, + ctas: cutlass.Int32, + budget_ns: cutlass.Int64, + ): + bidx, _, _ = cute.arch.block_idx() + storage = utils.SmemAllocator().allocate(self.shared_storage) + # Only shapes the partitions; prefetches write no shared memory. + box = cute.group_modes( + storage.box.get_tensor(cute.make_layout((128, 128), stride=(128, 1))), 0, 2 + ) + partitions = [] + for atom, tensor in ((atom_a, mA), (atom_b, mB)): + tiles = cute.local_tile(tensor, (128, 128), (None, None)) + partitions.append( + cpasync.tma_partition( + atom, 0, cute.make_layout(1), box, cute.group_modes(tiles, 0, 2) + )[1] + ) + tiles_a = self.shapes[0][0] // 128 + tiles_b = self.shapes[1][0] // 128 + units_a = tiles_a * (self.shapes[0][1] // 128) + units = units_a + tiles_b * (self.shapes[1][1] // 128) + count = (units - bidx + ctas - 1) // ctas + with cute.arch.elect_one(): + start = _globaltimer() + for group in cutlass.range((count + 15) // 16, unroll=1): + if _globaltimer() - start < budget_ns: + for j in cutlass.range_constexpr(16): + i = group * 16 + j + if i < count: + unit = bidx + i * ctas + if unit < units_a: + cute.prefetch( + atom_a, partitions[0][(None, unit % tiles_a, unit // tiles_a)] + ) + else: + rest = unit - units_a + cute.prefetch( + atom_b, partitions[1][(None, rest % tiles_b, rest // tiles_b)] + ) diff --git a/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_ob_boundary.py b/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_ob_boundary.py new file mode 100644 index 000000000000..9c48fa041f2b --- /dev/null +++ b/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_ob_boundary.py @@ -0,0 +1,708 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Attention ``o_b``, its TP all-reduce and the mHC boundary in one launch, bit for bit the LowM +GEMM then ``MhcAllReduceBoundaryKernel``: ``o_b`` CTAs push tiles, token CTAs finish boundaries.""" + +from __future__ import annotations + +import cutlass +import cutlass.cute as cute +from cutlass import BFloat16, Float32, Int32, Int64, Uint8, Uint32, const_expr +from cutlass.utils import SmemAllocator + +from ..cute_ptx_utils import mma_sync +from ..lowm_fp8_mma_gemm import CHUNK, GROUP, N_TILE, _fp8x4_to_bf16x4_s2, _fp8x4_to_bf16x4_scaled +from ..utils import griddepcontrol_launch_dependents, griddepcontrol_wait +from .mhc_boundary import bf16x2_hi, bf16x2_lo, ldg_f32 +from .mhc_split_boundary import ( + _NEG_ZERO, + HC, + VEC, + MhcAllReduceBoundaryKernel, + _arrive_barrier_1, + _atom_release_add_if_u32, + _bf16x2_neg_zero_to_zero, + _fence_acq_rel_gpu, + _ld_volatile_s32, + _ld_volatile_s64, + _ld_volatile_v4, + _st_v4, +) + + +def _round16(nbytes: int) -> int: + return (nbytes + 15) // 16 * 16 + + +def ob_boundary_smem_bytes( + hidden: int, k_warps: int, ctas_per_token: int = 2, n_tiles: int = 2, mbar_exchange: bool = True +) -> int: + """Dynamic shared memory of one CTA: the allocations of both roles, which do not overlap.""" + warps = hidden // VEC // ctas_per_token // 32 + groups = (warps - 1) // k_warps + cols_group = N_TILE * n_tiles + ob_role = ( + _round16(groups * k_warps * 16 * cols_group * 4) + + _round16(16 * cols_group * groups * 2) + + 16 + ) + token_role = _round16(warps * 4) + (16 if mbar_exchange else 0) + if ctas_per_token > 1: + token_role += _round16(ctas_per_token * warps * 4) + return ob_role + token_role + 128 + + +def ob_boundary_tiles( + hidden: int, + k_extent: int, + k_warps: int, + smem_limit: int, + ctas_per_token: int = 2, + n_tiles: int = 2, +) -> bool: + """Whether :class:`ObAllReduceBoundaryKernel` tiles an ``o_b`` of ``hidden`` x ``k_extent`` (the + constructors' shape checks) within ``smem_limit`` bytes of shared memory per CTA.""" + threads = hidden // VEC + if hidden % (32 * VEC) or threads > 1024 or threads % (32 * ctas_per_token): + return False + if k_warps < 1 or k_extent % (CHUNK * k_warps): + return False + groups = (threads // ctas_per_token // 32 - 1) // k_warps + cols_cta = N_TILE * n_tiles * groups + return ( + groups > 0 + and hidden % cols_cta == 0 + and (hidden // cols_cta) % ctas_per_token == 0 + and ob_boundary_smem_bytes(hidden, k_warps, ctas_per_token, n_tiles) <= smem_limit + ) + + +class ObAllReduceBoundaryKernel(MhcAllReduceBoundaryKernel): + """``o_b`` + one-shot all-reduce + mHC boundary of ``num_rows`` (<= 16) tokens: A [M, K] BF16, B + [H, K] E4M3, S [H, K/32] UE8M0 (linear); the rest as :class:`MhcAllReduceBoundaryKernel`. + + With ``mnnvl`` an ``o_b`` CTA also stores each pushed vector into this rank's own slot, so the + token CTAs here need not wait for its multicast copy.""" + + def __init__( + self, + *, + hidden: int, + k_extent: int, + num_rows: int, + k_warps: int, + norm_eps: float, + nranks: int, + use_pdl: bool, + ctas_per_token: int = 2, + n_tiles: int = 2, + mnnvl: bool = False, + quant: str | None = None, + mbar_exchange: bool = False, + bf16_dequant: bool = False, + ): + super().__init__( + hidden=hidden, + norm_eps=norm_eps, + nranks=nranks, + use_pdl=use_pdl, + ctas_per_token=ctas_per_token, + mnnvl=mnnvl, + quant=quant, + mbar_exchange=mbar_exchange, + ) + if not 1 <= num_rows <= 16: + raise ValueError(f"num_rows={num_rows} must be in [1, 16]") + if k_extent % (CHUNK * k_warps): + raise ValueError(f"K={k_extent} must be a multiple of {CHUNK * k_warps}") + if n_tiles not in (1, 2, 4): + raise ValueError(f"n_tiles={n_tiles} must be 1, 2 or 4") + warps = self.threads_cta // 32 + if warps - 1 < k_warps: + raise ValueError( + f"{warps} warps per CTA cannot hold k_warps={k_warps} + a control warp" + ) + self.k_extent = k_extent + self.num_rows = num_rows + # K splits over k_warps as in the GEMM kernel (the bits depend on it, not on n_tiles); an + # o_b CTA holds ``groups`` of k_warps warps and a control warp for the Lamport state. + self.k_warps = k_warps + self.chunks = k_extent // (CHUNK * k_warps) + self.n_tiles = n_tiles + self.bf16_dequant = bf16_dequant + self.groups = (warps - 1) // k_warps + self.cols_group = N_TILE * n_tiles + self.cols_cta = self.cols_group * self.groups + if hidden % self.cols_cta or (hidden // self.cols_cta) % ctas_per_token: + raise ValueError( + f"hidden={hidden} must split into whole clusters of {ctas_per_token} CTAs of " + f"{self.cols_cta} columns" + ) + self.ob_ctas = hidden // self.cols_cta + self.grid = self.ob_ctas + num_rows * ctas_per_token + self.smem_bytes = ob_boundary_smem_bytes( + hidden, k_warps, ctas_per_token, n_tiles, self.mbar_exchange + ) + + @cute.jit + def __call__( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gS: cute.Tensor, + gRes: cute.Tensor, + gPost: cute.Tensor, + gComb: cute.Tensor, + gPre: cute.Tensor, + gW: cute.Tensor, + gResOut: cute.Tensor, + gOut: cute.Tensor, + gWs: cute.Tensor, + rank: Int32, + q_addr: Int64, + sf_addr: Int64, + scale_addr: Int64, + stream, + ) -> None: + copy_a = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + BFloat16, + num_bits_per_copy=128, + load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS, + ) + copy_b = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + Uint8, + num_bits_per_copy=128, + load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING, + ) + copy_f32 = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + Float32, + num_bits_per_copy=128, + load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS, + ) + store_bf16 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=128 + ) + self.ob_kernel( + gA, + gB, + gS, + gRes, + gPost, + gComb, + gPre, + gW, + gResOut, + gOut, + gWs, + rank, + q_addr, + sf_addr, + scale_addr, + copy_a, + copy_b, + copy_f32, + store_bf16, + ).launch( + grid=[self.grid, 1, 1], + block=[self.threads_cta, 1, 1], + cluster=[self.ctas, 1, 1], + smem=self.smem_bytes, + stream=stream, + use_pdl=self.use_pdl, + ) + + @cute.kernel + def ob_kernel( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gS: cute.Tensor, + gRes: cute.Tensor, + gPost: cute.Tensor, + gComb: cute.Tensor, + gPre: cute.Tensor, + gW: cute.Tensor, + gResOut: cute.Tensor, + gOut: cute.Tensor, + gWs: cute.Tensor, + rank: Int32, + q_addr: Int64, + sf_addr: Int64, + scale_addr: Int64, + copy_a: cute.CopyAtom, + copy_b: cute.CopyAtom, + copy_f32: cute.CopyAtom, + store_bf16: cute.CopyAtom, + ) -> None: + tidx, _, _ = cute.arch.thread_idx() + bid, _, _ = cute.arch.block_idx() + if bid < self.ob_ctas: + self._ob_role(bid, tidx, gA, gB, gS, gWs, rank, copy_a, copy_b) + else: + self._token_role( + bid - self.ob_ctas, + tidx, + rank, + gRes, + gPost, + gComb, + gPre, + gW, + gResOut, + gOut, + gWs, + q_addr, + sf_addr, + scale_addr, + copy_a, + copy_f32, + store_bf16, + ) + + @cute.jit + def _ob_role( + self, + bid, + tidx, + gA: cute.Tensor, + gB: cute.Tensor, + gS: cute.Tensor, + gWs: cute.Tensor, + rank: Int32, + copy_a: cute.CopyAtom, + copy_b: cute.CopyAtom, + ) -> None: + """The GEMM tiles of o_b CTA ``bid``, pushed into every rank's Lamport slots.""" + warp = tidx // 32 + lane = tidx % 32 + r = lane // 4 + q = lane % 4 + num_rows: cutlass.Constexpr = self.num_rows + k_warps: cutlass.Constexpr = self.k_warps + chunks: cutlass.Constexpr = self.chunks + n_tiles: cutlass.Constexpr = self.n_tiles + groups: cutlass.Constexpr = self.groups + cols_g: cutlass.Constexpr = self.cols_group + cols: cutlass.Constexpr = self.cols_cta + nranks: cutlass.Constexpr = self.nranks + vecs: cutlass.Constexpr = self.hidden // VEC + tot: cutlass.Constexpr = self.num_rows * (self.hidden // VEC) + threads: cutlass.Constexpr = self.threads_cta + control_warp: cutlass.Constexpr = self.threads_cta // 32 - 1 + group = warp // k_warps + kw = warp % k_warps + n0 = bid * cols + group * cols_g + k_warp0 = kw * (chunks * CHUNK) + + # ---- weights + scales of the warp's tiles (independent of the producer kernel) ---- + gB_vec = cute.logical_divide(gB, (None, 16)) + b_regs = cute.make_rmem_tensor( + cute.make_layout((n_tiles * chunks, 16), stride=(16, 1)), Uint8 + ) + s_bits = cute.make_rmem_tensor(cute.make_layout((n_tiles * chunks,)), Int32) + s_regs = cute.recast_tensor(s_bits, Float32) + if group < groups: + for nt in cutlass.range_constexpr(n_tiles): + b_row = n0 + N_TILE * nt + r + for c in cutlass.range_constexpr(chunks): + k0 = k_warp0 + c * CHUNK + 16 * q + cute.copy( + copy_b, gB_vec[b_row, (None, k0 // 16)], b_regs[nt * chunks + c, None] + ) + # UE8M0 -> the bit pattern of 2^(s - 127). + s_bits[nt * chunks + c] = gS[b_row, k0 // GROUP].to(Int32) << 23 + # A UE8M0 scale's FP32 bits shifted by 16 are its BF16 bits. + s2 = cute.make_rmem_tensor(cute.make_layout((n_tiles * chunks,)), Uint32) + if const_expr(self.bf16_dequant): + for i in cutlass.range_constexpr(n_tiles * chunks): + s2[i] = Uint32(s_bits[i]) | (Uint32(s_bits[i]) >> 16) + # The workspace table holds addresses only, written by the host before any launch. + push_base = Int64(0) + own = Int64(0) + peers = cute.make_rmem_tensor(cute.make_layout((nranks,)), Int64) + if const_expr(self.mnnvl): + own = Int64(gWs[0]) + push_base = Int64(gWs[1]) + else: + for p in cutlass.range_constexpr(nranks): + peers[p] = Int64(gWs[p]) + + smem = SmemAllocator() + red = smem.allocate_tensor( + Float32, + cute.make_layout( + (groups, k_warps, 16, cols_g), + stride=(k_warps * 16 * cols_g, 16 * cols_g, cols_g, 1), + ), + byte_alignment=16, + ) + stage = smem.allocate_tensor( + BFloat16, cute.make_layout((16, cols), stride=(cols, 1)), byte_alignment=16 + ) + offs = smem.allocate_tensor(Int64, cute.make_layout((2,)), byte_alignment=16) + + if const_expr(self.use_pdl): + griddepcontrol_wait() + # The CTA whose arrival is counted last advances the Lamport state: no CTA waits for others. + state = Int32(0) + hand_flags = Int64(0) + hand_layout = Int64(0) + counter = Int64(0) + if warp == control_warp: + if lane == 0: + # This launch's Lamport buffer (the state is advanced only after every CTA arrived). + if const_expr(self.mnnvl): + flags = Int64(gWs[2]) + cur = _ld_volatile_s32(flags) + buf_bytes = Int64(_ld_volatile_s32(flags + Int64(8))) + offs[0] = Int64(cur) * buf_bytes + state = cur + hand_flags = flags + hand_layout = flags + counter = flags + Int64(32) + else: + flag_addr = Int64(gWs[nranks]) + layout_addr = Int64(gWs[nranks + 1]) + flag = _ld_volatile_s32(flag_addr + Int64(8)) + comm_size = _ld_volatile_s64(layout_addr + Int64(8)) + offs[0] = Int64(flag % 3) * comm_size + state = flag + hand_flags = flag_addr + hand_layout = layout_addr + counter = flag_addr + prior = _atom_release_add_if_u32(counter, Int32(1), Int32(tidx == control_warp * 32)) + + # ---- C tiles: the GEMM kernel's main loop (same K split, MMA order and FP32 sums) ---- + gA_vec = cute.logical_divide(gA, (None, 8)) + acc = cute.make_rmem_tensor(cute.make_layout((n_tiles, 4), stride=(4, 1)), Float32) + acc.fill(0.0) + a_lo0 = cute.make_rmem_tensor(cute.make_layout((8,)), BFloat16) + a_lo1 = cute.make_rmem_tensor(cute.make_layout((8,)), BFloat16) + a_hi0 = cute.make_rmem_tensor(cute.make_layout((8,)), BFloat16) + a_hi1 = cute.make_rmem_tensor(cute.make_layout((8,)), BFloat16) + a_lo0_u32 = cute.recast_tensor(a_lo0, Uint32) + a_lo1_u32 = cute.recast_tensor(a_lo1, Uint32) + a_hi0_u32 = cute.recast_tensor(a_hi0, Uint32) + a_hi1_u32 = cute.recast_tensor(a_hi1, Uint32) + a_frag = cute.make_rmem_tensor(cute.make_layout((8,)), BFloat16) + a_frag_u32 = cute.recast_tensor(a_frag, Uint32) + b_frag = cute.make_rmem_tensor(cute.make_layout((4,)), BFloat16) + b_frag_u32 = cute.recast_tensor(b_frag, Uint32) + tile = cute.make_rmem_tensor(cute.make_layout((4,)), Float32) + w_bf16 = cute.make_rmem_tensor(cute.make_layout((8,)), Uint32) + if group < groups: + for c in cutlass.range_constexpr(chunks): + k0 = k_warp0 + c * CHUNK + 16 * q + if const_expr(n_tiles == 1): + self._ob_convert(w_bf16, b_regs, s_regs, s2, c) + a_lo0_u32.fill(0) + a_lo1_u32.fill(0) + a_hi0_u32.fill(0) + a_hi1_u32.fill(0) + if r < num_rows: + cute.copy(copy_a, gA_vec[r, (None, k0 // 8)], a_lo0) + cute.copy(copy_a, gA_vec[r, (None, k0 // 8 + 1)], a_lo1) + if const_expr(8 < num_rows): + if r + 8 < num_rows: + cute.copy(copy_a, gA_vec[r + 8, (None, k0 // 8)], a_hi0) + cute.copy(copy_a, gA_vec[r + 8, (None, k0 // 8 + 1)], a_hi1) + for nt in cutlass.range_constexpr(n_tiles): + if const_expr(n_tiles > 1): + self._ob_convert(w_bf16, b_regs, s_regs, s2, nt * chunks + c) + for e in cutlass.range_constexpr(4): + tile[e] = acc[nt, e] + for j in cutlass.range_constexpr(4): + if const_expr(j < 2): + a_frag_u32[0] = a_lo0_u32[2 * j] + a_frag_u32[1] = a_hi0_u32[2 * j] + a_frag_u32[2] = a_lo0_u32[2 * j + 1] + a_frag_u32[3] = a_hi0_u32[2 * j + 1] + else: + a_frag_u32[0] = a_lo1_u32[2 * j - 4] + a_frag_u32[1] = a_hi1_u32[2 * j - 4] + a_frag_u32[2] = a_lo1_u32[2 * j - 3] + a_frag_u32[3] = a_hi1_u32[2 * j - 3] + b_frag_u32[0] = w_bf16[2 * j] + b_frag_u32[1] = w_bf16[2 * j + 1] + tile.store(mma_sync(a_frag, b_frag, tile)) + for e in cutlass.range_constexpr(4): + acc[nt, e] = tile[e] + + # ---- split-K sums in warp order, one BF16 rounding, staged per row ---- + if group < groups: + for nt in cutlass.range_constexpr(n_tiles): + c0 = N_TILE * nt + 2 * q + red[group, kw, r, c0] = acc[nt, 0] + red[group, kw, r, c0 + 1] = acc[nt, 1] + red[group, kw, r + 8, c0] = acc[nt, 2] + red[group, kw, r + 8, c0 + 1] = acc[nt, 3] + cute.arch.sync_threads() + for i in cutlass.range(tidx, groups * num_rows * cols_g, threads): + g = i // (num_rows * cols_g) + rem = i % (num_rows * cols_g) + row = rem // cols_g + col = rem % cols_g + total = Float32(0.0) + for w in cutlass.range_constexpr(k_warps): + total = total + red[g, w, row, col] + stage[row, g * cols_g + col] = total.to(BFloat16) + cute.arch.sync_threads() + data_off = offs[0] + + # ---- push: one 16-byte vector per (row, 8 columns), as the boundary kernel pushes it ---- + neg0 = Int32(_NEG_ZERO) + stage_v = cute.logical_divide(stage, (None, N_TILE)) + vals = cute.make_rmem_tensor(cute.make_layout((N_TILE,)), BFloat16) + words = cute.recast_tensor(vals, Int32) + tiles_cta: cutlass.Constexpr = cols // N_TILE + for v in cutlass.range(tidx, num_rows * tiles_cta, threads): + row = v // tiles_cta + nt = v % tiles_cta + vcol = bid * tiles_cta + nt + cute.autovec_copy(stage_v[row, (None, nt)], vals) + w0 = words[0] + w1 = words[1] + w2 = words[2] + w3 = words[3] + slot = (Int64(rank) * Int64(tot) + Int64(row) * Int64(vecs) + Int64(vcol)) * Int64(16) + if const_expr(self.mnnvl): + w0 = _bf16x2_neg_zero_to_zero(w0) + w1 = _bf16x2_neg_zero_to_zero(w1) + w2 = _bf16x2_neg_zero_to_zero(w2) + w3 = _bf16x2_neg_zero_to_zero(w3) + _st_v4(push_base + data_off + slot, w0, w1, w2, w3) + _st_v4(own + data_off + slot, w0, w1, w2, w3) + else: + if w0 == neg0: + w0 = Int32(0) + if w1 == neg0: + w1 = Int32(0) + if w2 == neg0: + w2 = Int32(0) + if w3 == neg0: + w3 = Int32(0) + for p in cutlass.range_constexpr(nranks): + _st_v4(peers[p] + data_off + slot, w0, w1, w2, w3) + if const_expr(self.use_pdl): + griddepcontrol_launch_dependents() + if warp == control_warp: + if lane == 0: + self._handover_if_last(prior, state, hand_flags, hand_layout) + + @cute.jit + def _handover_if_last(self, prior, state, flag_addr: Int64, layout_addr: Int64) -> None: + """The CTA whose arrival was counted last advances the Lamport state.""" + if prior == Int32(self.grid - 1): + _fence_acq_rel_gpu() + self._advance_state(state, self.num_rows, flag_addr, layout_addr) + + @cute.jit + def _ob_convert( + self, + w_bf16: cute.Tensor, + b_regs: cute.Tensor, + s_regs: cute.Tensor, + s2: cute.Tensor, + slot: cutlass.Constexpr, + ) -> None: + """The 16 E4M3 weights of register slot ``slot`` times their scale -> 8 BF16 pairs.""" + raw = cute.recast_tensor(b_regs[slot, None], Uint32) + for w in cutlass.range_constexpr(4): + if const_expr(self.bf16_dequant): + pair = _fp8x4_to_bf16x4_s2(raw[w], s2[slot]) + else: + pair = _fp8x4_to_bf16x4_scaled(raw[w], s_regs[slot]) + w_bf16[2 * w] = pair[0] + w_bf16[2 * w + 1] = pair[1] + + @cute.jit + def _token_role( + self, + b, + tidx, + rank: Int32, + gRes: cute.Tensor, + gPost: cute.Tensor, + gComb: cute.Tensor, + gPre: cute.Tensor, + gW: cute.Tensor, + gResOut: cute.Tensor, + gOut: cute.Tensor, + gWs: cute.Tensor, + q_addr: Int64, + sf_addr: Int64, + scale_addr: Int64, + copy_bf16: cute.CopyAtom, + copy_f32: cute.CopyAtom, + store_bf16: cute.CopyAtom, + ) -> None: + """The boundary of token ``b // ctas``: everything the boundary kernel does but the push.""" + ctas: cutlass.Constexpr = self.ctas + nranks: cutlass.Constexpr = self.nranks + vecs: cutlass.Constexpr = self.hidden // VEC + tot: cutlass.Constexpr = self.num_rows * (self.hidden // VEC) + tok = b // ctas + vi = (b % ctas) * self.threads_cta + tidx + neg0 = Int32(_NEG_ZERO) + + gRes_v = cute.logical_divide(gRes, (None, VEC)) + gResOut_v = cute.logical_divide(gResOut, (None, VEC)) + gOut_v = cute.logical_divide(gOut, (None, VEC)) + gW_v = cute.logical_divide(gW, (None, VEC)) + gPost_v = cute.logical_divide(gPost, (None, 4)) + gComb_v = cute.logical_divide(gComb, (None, 4)) + gPre_v = cute.logical_divide(gPre, (None, 4)) + + # ---- the norm weight and the host-written workspace addresses: before the wait ---- + w_bf = cute.make_rmem_tensor(cute.make_layout((VEC,)), BFloat16) + cute.copy(copy_bf16, gW_v[0, (None, vi)], w_bf) + gscale = Float32(0.0) + if const_expr(self.quant == "nvfp4"): + gscale = ldg_f32(scale_addr) + own = Int64(0) + flags = Int64(0) + flag_addr = Int64(0) + layout_addr = Int64(0) + if const_expr(self.mnnvl): + own = Int64(gWs[0]) + flags = Int64(gWs[2]) + else: + flag_addr = Int64(gWs[nranks]) + layout_addr = Int64(gWs[nranks + 1]) + own = Int64(gWs[rank]) + mbar_addr = Int32(0) + if const_expr(self.mbar_exchange): + mbar_addr = self._mbar_setup(tidx) + + if const_expr(self.use_pdl): + griddepcontrol_wait() + # The wait covers the coefficient kernel too, a second prerequisite grid when it releases early. + pre, post, comb = self._load_coefficients(tok, gPost_v, gComb_v, gPre_v, copy_f32) + + # ---- Lamport state of this launch (read before any CTA bumps the arrival counter) ---- + stage_sizes = cute.make_rmem_tensor(cute.make_layout((4,)), Int32) + cur = Int32(0) + dirty = Int32(0) + buf_bytes = Int64(0) + dirty_stages = Int32(0) + flag = Int32(0) + clear_size = Int64(0) + clear_off = Int64(0) + data_off = Int64(0) + if const_expr(self.mnnvl): + cur = _ld_volatile_s32(flags) + dirty = _ld_volatile_s32(flags + Int64(4)) + buf_bytes = Int64(_ld_volatile_s32(flags + Int64(8))) + dirty_stages = _ld_volatile_s32(flags + Int64(12)) + for s in cutlass.range_constexpr(4): + stage_sizes[s] = _ld_volatile_s32(flags + Int64(16 + 4 * s)) + data_off = Int64(cur) * buf_bytes + if tidx < 32: + _arrive_barrier_1(Int32(self.threads_cta), wait=True) + else: + _arrive_barrier_1(Int32(self.threads_cta), wait=False) + else: + flag = _ld_volatile_s32(flag_addr + Int64(8)) + clear_size = _ld_volatile_s64(layout_addr) + comm_size = _ld_volatile_s64(layout_addr + Int64(8)) + data_off = Int64(flag % 3) * comm_size + clear_off = Int64((flag + 2) % 3) * comm_size + if tidx < 32: + _arrive_barrier_1(Int32(self.threads_cta), wait=True) + else: + _arrive_barrier_1(Int32(self.threads_cta), wait=False) + # Thread 0 counts the CTA after warp 0's barrier wait. + counter = flag_addr + if const_expr(self.mnnvl): + counter = flags + Int64(32) + prior = _atom_release_add_if_u32(counter, Int32(1), Int32(tidx == 0)) + if const_expr(self.use_pdl): + griddepcontrol_launch_dependents() + # After the wait: launches since the residual's boundary may release before their own wait. + # After the arrival too, whose release would otherwise wait for these loads. + r_bf = cute.make_rmem_tensor(cute.make_layout((HC, VEC), stride=(VEC, 1)), BFloat16) + for j in cutlass.range_constexpr(HC): + cute.copy(copy_bf16, gRes_v[tok, (None, j * vecs + vi)], r_bf[j, None]) + + idx = Int64(tok) * Int64(vecs) + Int64(vi) + # ---- clear the slots the previous launch used ---- + if const_expr(self.mnnvl): + for s in cutlass.range_constexpr(4): + if Int32(s) < dirty_stages: + stage_bytes = buf_bytes // Int64(dirty_stages) + stage = own + (Int64(dirty) * Int64(dirty_stages) + Int64(s)) * stage_bytes + clear_vecs = Int64(stage_sizes[s]) // Int64(16) + c = idx + while c < clear_vecs: + _st_v4(stage + c * Int64(16), neg0, neg0, neg0, neg0) + c = c + Int64(tot) + else: + # Its size in bytes (every Lamport kernel on the workspace), as whole 16-byte stores. + clear_vecs = (clear_size + Int64(15)) // Int64(16) + c = idx + while c < clear_vecs: + _st_v4(own + clear_off + c * Int64(16), neg0, neg0, neg0, neg0) + c = c + Int64(tot) + + if tidx == 0: + if const_expr(self.mnnvl): + self._handover_if_last(prior, cur, flags, flags) + else: + self._handover_if_last(prior, flag, flag_addr, layout_addr) + + # ---- wait for the token's slice of every rank (pushed by the o_b CTAs of every rank) ---- + got = cute.make_rmem_tensor(cute.make_layout((nranks, 4), stride=(4, 1)), Int32) + pending = Int32(1) + while pending != 0: + pending = Int32(0) + for p in cutlass.range_constexpr(nranks): + a, bb, cc, d = _ld_volatile_v4( + own + data_off + (Int64(p) * Int64(tot) + idx) * Int64(16) + ) + got[p, 0] = a + got[p, 1] = bb + got[p, 2] = cc + got[p, 3] = d + if a == neg0: + pending = Int32(1) + if bb == neg0: + pending = Int32(1) + if cc == neg0: + pending = Int32(1) + if d == neg0: + pending = Int32(1) + + # ---- x = BF16(FP32 sum in rank order), as the library kernel's FP32 accumulation ---- + x_bf = cute.make_rmem_tensor(cute.make_layout((VEC,)), BFloat16) + for i in cutlass.range_constexpr(4): + lo = bf16x2_lo(got[0, i]) + hi = bf16x2_hi(got[0, i]) + for p in cutlass.range_constexpr(1, nranks): + lo = lo + bf16x2_lo(got[p, i]) + hi = hi + bf16x2_hi(got[p, i]) + x_bf[2 * i] = lo.to(BFloat16) + x_bf[2 * i + 1] = hi.to(BFloat16) + + self._boundary( + tok, + tidx, + vi, + ctas, + x_bf, + r_bf, + w_bf, + pre, + post, + comb, + gResOut_v, + gOut_v, + store_bf16, + False, + q_addr, + sf_addr, + gscale, + mbar_addr, + ) diff --git a/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_split_boundary.py b/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_split_boundary.py index 80a116c2361f..75a0bc236e8e 100644 --- a/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_split_boundary.py +++ b/tensorrt_llm/_torch/kernels/blackwell/mhc/mhc_split_boundary.py @@ -143,6 +143,56 @@ def _ld_volatile_v4(addr: Int64, *, loc=None, ip=None): return tuple(Int32(llvm.extractvalue(T.i32(), words, [i], loc=loc, ip=ip)) for i in range(4)) +@dsl_user_op +def _ld_volatile_v4_or( + addr: Int64, use_words: Int32, w0: Int32, w1: Int32, w2: Int32, w3: Int32, *, loc=None, ip=None +): + """``_ld_volatile_v4`` of ``addr``, or ``w0..w3`` without a load where ``use_words`` != 0.""" + words = _asm( + ir.Type.parse("!llvm.struct<(i32,i32,i32,i32)>"), + [Int64(addr).ir_value(loc=loc, ip=ip), Int32(use_words).ir_value(loc=loc, ip=ip)] + + [Int32(w).ir_value(loc=loc, ip=ip) for w in (w0, w1, w2, w3)], + "{\n\t.reg .pred p;\n\tsetp.ne.s32 p, $5, 0;\n\t" + "@!p ld.volatile.global.v4.b32 {$0, $1, $2, $3}, [$4];\n\t" + "@p mov.b32 $0, $6;\n\t@p mov.b32 $1, $7;\n\t@p mov.b32 $2, $8;\n\t@p mov.b32 $3, $9;\n\t}", + "=&r,=&r,=&r,=&r,l,r,r,r,r,r,~{memory}", + loc=loc, + ip=ip, + ) + return tuple(Int32(llvm.extractvalue(T.i32(), words, [i], loc=loc, ip=ip)) for i in range(4)) + + +@dsl_user_op +def _ld_relaxed_s32(addr: Int64, *, loc=None, ip=None) -> Int32: + """A gpu-scope relaxed load: the L2 value, never an L1 copy.""" + return Int32( + _asm( + T.i32(), + [Int64(addr).ir_value(loc=loc, ip=ip)], + "ld.relaxed.gpu.global.b32 $0, [$1];", + "=r,l", + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _ld_relaxed_bf16_as_f32(addr: Int64, *, loc=None, ip=None) -> Float32: + """The BF16 at ``addr`` (gpu-scope relaxed load) as FP32, exactly.""" + return Float32( + _asm( + T.f32(), + [Int64(addr).ir_value(loc=loc, ip=ip)], + "{\n\t.reg .b16 h, z;\n\t.reg .b32 w;\n\tld.relaxed.gpu.global.b16 h, [$1];\n\t" + "mov.b16 z, 0;\n\tmov.b32 w, {z, h};\n\tmov.b32 $0, w;\n\t}", + "=f,l", + loc=loc, + ip=ip, + ) + ) + + @dsl_user_op def _st_v4(addr: Int64, w0: Int32, w1: Int32, w2: Int32, w3: Int32, *, loc=None, ip=None) -> None: _asm( @@ -156,6 +206,18 @@ def _st_v4(addr: Int64, w0: Int32, w1: Int32, w2: Int32, w3: Int32, *, loc=None, ) +@dsl_user_op +def _prefetch_l2_bulk(addr: Int64, nbytes: Int32, *, loc=None, ip=None) -> None: + _asm( + None, + [Int64(addr).ir_value(loc=loc, ip=ip), Int32(nbytes).ir_value(loc=loc, ip=ip)], + "cp.async.bulk.prefetch.L2.global [$0], $1;", + "l,r", + loc=loc, + ip=ip, + ) + + @dsl_user_op def _ld_volatile_s32(addr: Int64, *, loc=None, ip=None) -> Int32: return Int32( @@ -247,6 +309,32 @@ def _red_async_release_add_u32(addr: Int64, value: Int32, *, loc=None, ip=None) ) +@dsl_user_op +def _atom_release_add_if_u32(addr: Int64, value: Int32, pred: Int32, *, loc=None, ip=None) -> Int32: + """``atom.release.gpu`` add returning the previous value where ``pred`` != 0 (-1 elsewhere), in + one predicated asm statement so that no branch merge waits for its return.""" + return Int32( + _asm( + T.i32(), + [ + Int64(addr).ir_value(loc=loc, ip=ip), + Int32(value).ir_value(loc=loc, ip=ip), + Int32(pred).ir_value(loc=loc, ip=ip), + ], + "{\n\t.reg .pred p;\n\tsetp.ne.s32 p, $3, 0;\n\tmov.u32 $0, 0xFFFFFFFF;\n\t" + "@p atom.release.gpu.global.add.u32 $0, [$1], $2;\n\t}", + "=&r,l,r,r,~{memory}", + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _fence_acq_rel_gpu(*, loc=None, ip=None) -> None: + _asm(None, [], "fence.acq_rel.gpu;", "~{memory}", loc=loc, ip=ip) + + @dsl_user_op def _arrive_barrier_1(threads: Int32, *, wait: bool, loc=None, ip=None) -> None: """Named barrier 1 over ``threads`` threads: wait for every arrival, or only arrive.""" @@ -302,6 +390,87 @@ def _st_dsmem_f32(smem_addr: Int32, cta_rank: Int32, value: Float32, *, loc=None ) +@dsl_user_op +def _mbar_init(addr: Int32, count: Int32, *, loc=None, ip=None) -> None: + _asm( + None, + [Int32(addr).ir_value(loc=loc, ip=ip), Int32(count).ir_value(loc=loc, ip=ip)], + "mbarrier.init.shared::cta.b64 [$0], $1;", + "r,r,~{memory}", + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _fence_mbar_init_cluster(*, loc=None, ip=None) -> None: + _asm(None, [], "fence.mbarrier_init.release.cluster;", "~{memory}", loc=loc, ip=ip) + + +@dsl_user_op +def _cluster_arrive_relaxed_wait(*, loc=None, ip=None) -> None: + _asm( + None, + [], + "barrier.cluster.arrive.relaxed.aligned;\n\tbarrier.cluster.wait.aligned;", + "~{memory}", + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _mbar_expect_tx(addr: Int32, nbytes: Int32, *, loc=None, ip=None) -> None: + _asm( + None, + [Int32(addr).ir_value(loc=loc, ip=ip), Int32(nbytes).ir_value(loc=loc, ip=ip)], + "mbarrier.arrive.expect_tx.shared::cta.b64 _, [$0], $1;", + "r,r,~{memory}", + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def _mbar_try_wait(addr: Int32, parity: Int32, *, loc=None, ip=None) -> Int32: + return Int32( + _asm( + T.i32(), + [Int32(addr).ir_value(loc=loc, ip=ip), Int32(parity).ir_value(loc=loc, ip=ip)], + # Cluster-scope acquire: the completions come from peer CTAs' st.async. + "{\n\t.reg .pred p;\n\t" + "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 p, [$1], $2;\n\t" + "selp.b32 $0, 1, 0, p;\n\t}", + "=r,r,r,~{memory}", + loc=loc, + ip=ip, + ) + ) + + +@dsl_user_op +def _st_async_dsmem_f32( + slot: Int32, mbar: Int32, cta_rank: Int32, value: Float32, *, loc=None, ip=None +) -> None: + """``st.async`` of FP32 ``value`` to shared address ``slot`` of cluster CTA ``cta_rank``, + completing 4 bytes on that CTA's mbarrier at shared address ``mbar``.""" + _asm( + None, + [ + Int32(slot).ir_value(loc=loc, ip=ip), + Int32(mbar).ir_value(loc=loc, ip=ip), + Int32(cta_rank).ir_value(loc=loc, ip=ip), + Float32(value).ir_value(loc=loc, ip=ip), + ], + "{\n\t.reg .b32 ra, rb, rv;\n\tmapa.shared::cluster.u32 ra, $0, $2;\n\t" + "mapa.shared::cluster.u32 rb, $1, $2;\n\tmov.b32 rv, $3;\n\t" + "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [ra], rv, [rb];\n\t}", + "r,r,r,f,~{memory}", + loc=loc, + ip=ip, + ) + + # A 32-bit Lamport slot holding -0.0 has not been written yet. _NEG_ZERO = -2147483648 # 0x80000000 as int32 @@ -795,6 +964,7 @@ def _boundary( q_addr: Int64, sf_addr: Int64, gscale: Float32, + mbar_addr=None, ) -> None: """Everything after ``x`` is known: post update, residual store, collapse, RMSNorm. @@ -858,6 +1028,7 @@ def _boundary( q_addr, sf_addr, gscale, + mbar_addr, ) @cute.jit @@ -875,6 +1046,7 @@ def _collapse_norm( q_addr: Int64, sf_addr: Int64, gscale: Float32, + mbar_addr=None, ) -> None: warp = tidx // 32 lane = tidx % 32 @@ -903,18 +1075,31 @@ def _collapse_norm( total = total + red[w] else: # Every CTA holds all ctas x num_warps partials: lane 0 of each warp pushes its partial - # into slot (this CTA, warp) of every CTA, one cluster barrier publishes them, and the - # sum runs over local shared memory in the same order. + # into slot (this CTA, warp) of every CTA, and the sum runs over local shared memory in + # the same order. all_red = smem.allocate_tensor( Float32, cute.make_layout((ctas * num_warps,)), byte_alignment=16 ) me = (vi - tidx) // (num_warps * 32) - if lane == 0: - slot_addr = Int32(all_red.iterator.toint()) + Int32(4) * (me * num_warps + warp) - for p in cutlass.range_constexpr(ctas): - _st_dsmem_f32(slot_addr, Int32(p), sum_sq) - cute.arch.cluster_arrive() - cute.arch.cluster_wait() + slot_addr = Int32(all_red.iterator.toint()) + Int32(4) * (me * num_warps + warp) + if const_expr(self.mbar_exchange): + # The partials complete on this CTA's mbarrier at ``mbar_addr`` (from _mbar_setup, + # before the dependency wait), so no cluster barrier waits for the residual stores. + if tidx == 0: + _mbar_expect_tx(mbar_addr, Int32(4 * ctas * num_warps)) + if lane == 0: + for p in cutlass.range_constexpr(ctas): + _st_async_dsmem_f32(slot_addr, mbar_addr, Int32(p), sum_sq) + done = _mbar_try_wait(mbar_addr, Int32(0)) + while done == 0: + done = _mbar_try_wait(mbar_addr, Int32(0)) + else: + # One cluster barrier publishes the partials. + if lane == 0: + for p in cutlass.range_constexpr(ctas): + _st_dsmem_f32(slot_addr, Int32(p), sum_sq) + cute.arch.cluster_arrive() + cute.arch.cluster_wait() for p in cutlass.range_constexpr(ctas): for w in cutlass.range_constexpr(num_warps): total = total + all_red[p * num_warps + w] @@ -995,6 +1180,17 @@ class MhcAllReduceBoundaryKernel(MhcPostCollapseNormKernel): output (as TensorRT-LLM's MoE-finalize all-reduce): ``sum_k scale[t, k] * fc2[perm[t, k]]`` plus the shared-expert output, accumulated in FP32 and rounded to BF16 once (``perm < 0`` rows are skipped). ``gX`` is then unused; without it ``gFc2`` / ``gScale`` / ``gPerm`` / ``gShared`` are. + + Same sums in the same order with ``mbar_exchange`` (cluster CTAs exchange RMSNorm partials by + ``st.async`` on a per-CTA mbarrier) and ``early_moe_rows`` (MoE rows load before the flags). + ``early_clear`` (MNNVL, MoE-finalize input): the Lamport state and the routing are read and the + previous buffer cleared before the dependency wait; the caller guarantees the previous launch has + completed and the routing is final. ``own_slot``: this rank's slot is taken from the words the + thread pushed instead of being polled. ``handover_warp`` (with ``mbar_exchange``): an extra warp + per CTA counts the CTA once its threads have read the Lamport state, and the CTA counted last + hands the state over, so no thread waits for the others' arrivals. ``l2_prefetch`` (with the + hand-over warp): that warp also reads up to two given regions into L2 once the dependency wait + returns, the CTAs splitting them. """ def __init__( @@ -1008,6 +1204,12 @@ def __init__( ctas_per_token: int = 1, mnnvl: bool = False, quant: str | None = None, + mbar_exchange: bool = False, + early_moe_rows: bool = False, + early_clear: bool = False, + own_slot: bool = False, + handover_warp: bool = False, + l2_prefetch: bool = False, ): super().__init__( hidden=hidden, norm_eps=norm_eps, use_pdl=use_pdl, post_update=True, quant=quant @@ -1022,6 +1224,12 @@ def __init__( # A cluster of CTAs per token spreads the token's pushes, polls and math over SMs. self.ctas = ctas_per_token self.threads_cta = self.threads // ctas_per_token + self.mbar_exchange = mbar_exchange and ctas_per_token > 1 + self.early_moe_rows = early_moe_rows and moe_topk > 0 + self.early_clear = early_clear and mnnvl and moe_topk > 0 + self.own_slot = own_slot + self.handover_warp = handover_warp and self.mbar_exchange + self.l2_prefetch = l2_prefetch and self.handover_warp @cute.jit def __call__( @@ -1043,6 +1251,10 @@ def __call__( q_addr: Int64, sf_addr: Int64, scale_addr: Int64, + pf0_addr: Int64, + pf0_bytes: Int64, + pf1_addr: Int64, + pf1_bytes: Int64, stream, ) -> None: m = cute.size(gRes, mode=[0]) @@ -1079,20 +1291,259 @@ def __call__( q_addr, sf_addr, scale_addr, + pf0_addr, + pf0_bytes, + pf1_addr, + pf1_bytes, copy_bf16, copy_f32, store_bf16, ).launch( grid=[m * self.ctas, 1, 1], - block=[self.threads_cta, 1, 1], + block=[self.threads_cta + (32 if self.handover_warp else 0), 1, 1], cluster=[self.ctas, 1, 1], - smem=self.num_warps * 4 * (2 if self.ctas > 1 else 1), + smem=self.num_warps * 4 * (2 if self.ctas > 1 else 1) + + (64 if self.mbar_exchange else 0), stream=stream, use_pdl=self.use_pdl, ) + @cute.jit + def _mbar_setup(self, tidx): + """This CTA's RMSNorm-partials mbarrier, initialized and cluster-visible before any peer's + ``st.async``; returns its shared address. Reads no producer data: runs before the wait.""" + smem = SmemAllocator() + mbar = smem.allocate_tensor(Int64, cute.make_layout((1,)), byte_alignment=16) + addr = Int32(mbar.iterator.toint()) + if tidx == 0: + _mbar_init(addr, Int32(1)) + _fence_mbar_init_cluster() + _cluster_arrive_relaxed_wait() + return addr + + @cute.jit + def _mnnvl_state(self, flags: Int64): + cur = _ld_volatile_s32(flags) + dirty = _ld_volatile_s32(flags + Int64(4)) + buf_bytes = Int64(_ld_volatile_s32(flags + Int64(8))) + dirty_stages = _ld_volatile_s32(flags + Int64(12)) + stage_sizes = cute.make_rmem_tensor(cute.make_layout((4,)), Int32) + for s in cutlass.range_constexpr(4): + stage_sizes[s] = _ld_volatile_s32(flags + Int64(16 + 4 * s)) + return cur, dirty, buf_bytes, dirty_stages, stage_sizes + + @cute.jit + def _advance_state(self, state, m, flag_addr: Int64, layout_addr: Int64) -> None: + """Hand the triple buffer to the next launch once every CTA has read the state (``state``: + MNNVL the current buffer index, IPC the flag value).""" + nranks: cutlass.Constexpr = self.nranks + if const_expr(self.mnnvl): + # This launch's buffer becomes the dirty one (one stage, all ranks' slices). + _st_s32(flag_addr, (state + 1) % 3) + _st_s32(flag_addr + Int64(4), state) + _st_s32(flag_addr + Int64(12), Int32(1)) + _st_s32(flag_addr + Int64(16), Int32(m) * Int32(self.hidden * nranks * 2)) + _st_s32(flag_addr + Int64(20), Int32(0)) + _st_s32(flag_addr + Int64(24), Int32(0)) + _st_s32(flag_addr + Int64(28), Int32(0)) + _st_s32(flag_addr + Int64(32), Int32(0)) + else: + _st_s32(flag_addr + Int64(8), (state + 1) % 3) + _st_s64(layout_addr, Int64(m) * Int64(self.hidden * nranks * 2)) + _st_s32(flag_addr, Int32(0)) + + @cute.jit + def _clear_mnnvl_dirty( + self, + own: Int64, + dirty: Int32, + buf_bytes: Int64, + dirty_stages: Int32, + stage_sizes: cute.Tensor, + idx: Int64, + tot: Int64, + ) -> None: + """This thread's share of the clear of the previous launch's (dirty) buffer.""" + neg0 = Int32(_NEG_ZERO) + for s in cutlass.range_constexpr(4): + if Int32(s) < dirty_stages: + stage_bytes = buf_bytes // Int64(dirty_stages) + stage = own + (Int64(dirty) * Int64(dirty_stages) + Int64(s)) * stage_bytes + clear_vecs = Int64(stage_sizes[s]) // Int64(16) + c = idx + while c < clear_vecs: + _st_v4(stage + c * Int64(16), neg0, neg0, neg0, neg0) + c = c + tot + + @cute.jit + def _load_routing( + self, + tok, + gPerm: cute.Tensor, + gScale: cute.Tensor, + perm: cute.Tensor, + weight: cute.Tensor, + before_wait: cutlass.Constexpr, + ) -> None: + """The token's expert rows ``perm`` and routing weights ``weight`` (FP32); ``before_wait`` + reads them from L2, as no L1 line is known to be fresh before the dependency wait.""" + for k in cutlass.range_constexpr(self.moe_topk): + if const_expr(before_wait): + perm[k] = _ld_relaxed_s32( + gPerm.iterator.toint() + (Int64(tok) * self.moe_topk + k) * 4 + ) + weight[k] = _ld_relaxed_bf16_as_f32( + gScale.iterator.toint() + (Int64(tok) * self.moe_topk + k) * 2 + ) + else: + perm[k] = Int32(gPerm[tok, k]) + weight[k] = gScale[tok, k].to(Float32) + + @cute.jit + def _load_moe_rows( + self, + tok, + vi, + perm: cute.Tensor, + gFc2_v: cute.Tensor, + gShared_v: cute.Tensor, + rows: cute.Tensor, + shared: cute.Tensor, + copy_bf16: cute.CopyAtom, + ) -> None: + """The token's routed FC2 rows and shared-expert row for vector ``vi``; all rows are loaded + before the sum (a skipped row, ``perm < 0``, reads row 0 unused).""" + for k in cutlass.range_constexpr(self.moe_topk): + p = perm[k] + if p < 0: + p = Int32(0) + cute.copy(copy_bf16, gFc2_v[p, (None, vi)], rows[k, None]) + cute.copy(copy_bf16, gShared_v[tok, (None, vi)], shared) + @cute.kernel def ar_kernel( + self, + gX: cute.Tensor, + gRes: cute.Tensor, + gPost: cute.Tensor, + gComb: cute.Tensor, + gPre: cute.Tensor, + gW: cute.Tensor, + gResOut: cute.Tensor, + gOut: cute.Tensor, + gWs: cute.Tensor, + rank: Int32, + gFc2: cute.Tensor, + gScale: cute.Tensor, + gPerm: cute.Tensor, + gShared: cute.Tensor, + q_addr: Int64, + sf_addr: Int64, + scale_addr: Int64, + pf0_addr: Int64, + pf0_bytes: Int64, + pf1_addr: Int64, + pf1_bytes: Int64, + copy_bf16: cute.CopyAtom, + copy_f32: cute.CopyAtom, + store_bf16: cute.CopyAtom, + ) -> None: + tidx, _, _ = cute.arch.thread_idx() + token = True + if const_expr(self.handover_warp): + token = tidx < self.threads_cta + if token: + self._ar_token( + gX, + gRes, + gPost, + gComb, + gPre, + gW, + gResOut, + gOut, + gWs, + rank, + gFc2, + gScale, + gPerm, + gShared, + q_addr, + sf_addr, + scale_addr, + copy_bf16, + copy_f32, + store_bf16, + ) + else: + self._handover_warp(tidx, gWs, pf0_addr, pf0_bytes, pf1_addr, pf1_bytes) + + @cute.jit + def _prefetch_l2_slice(self, lane, addr: Int64, nbytes: Int64) -> None: + """This CTA's share of ``nbytes`` (a multiple of 16) at ``addr``: 16 KB per lane at a time.""" + grid, _, _ = cute.arch.grid_dim() + bid, _, _ = cute.arch.block_idx() + share = (nbytes // Int64(16) + Int64(grid) - Int64(1)) // Int64(grid) * Int64(16) + start = Int64(bid) * share + end = start + share + if end > nbytes: + end = nbytes + off = start + Int64(lane) * Int64(16384) + while off < end: + n = end - off + if n > Int64(16384): + n = Int64(16384) + _prefetch_l2_bulk(addr + off, Int32(n)) + off = off + Int64(32 * 16384) + + @cute.jit + def _handover_warp( + self, + tidx, + gWs: cute.Tensor, + pf0_addr: Int64, + pf0_bytes: Int64, + pf1_addr: Int64, + pf1_bytes: Int64, + ) -> None: + """Count the CTA once its token threads have read the Lamport state; the CTA counted last + advances the state. With ``l2_prefetch`` it first issues the CTA's prefetches.""" + grid, _, _ = cute.arch.grid_dim() + nranks: cutlass.Constexpr = self.nranks + # Every thread of the cluster takes part in the barrier of the token threads' _mbar_setup. + _cluster_arrive_relaxed_wait() + if const_expr(self.l2_prefetch): + # After the wait: the producer's own weight reads are done by then. + if const_expr(self.use_pdl): + griddepcontrol_wait() + self._prefetch_l2_slice(tidx - self.threads_cta, pf0_addr, pf0_bytes) + self._prefetch_l2_slice(tidx - self.threads_cta, pf1_addr, pf1_bytes) + flag_addr = Int64(0) + layout_addr = Int64(0) + counter = Int64(0) + if const_expr(self.mnnvl): + flag_addr = Int64(gWs[2]) + layout_addr = flag_addr + counter = flag_addr + Int64(32) + else: + flag_addr = Int64(gWs[nranks]) + layout_addr = Int64(gWs[nranks + 1]) + counter = flag_addr + _arrive_barrier_1(Int32(self.threads_cta + 32), wait=True) + # The state the token threads read (no launch advances it before every CTA is counted). + state = Int32(0) + if const_expr(self.mnnvl): + state = _ld_volatile_s32(flag_addr) + else: + state = _ld_volatile_s32(flag_addr + Int64(8)) + prior = _atom_release_add_if_u32(counter, Int32(1), Int32(tidx == self.threads_cta)) + if tidx == self.threads_cta: + if prior == grid - 1: + _fence_acq_rel_gpu() + self._advance_state(state, grid // self.ctas, flag_addr, layout_addr) + + @cute.jit + def _ar_token( self, gX: cute.Tensor, gRes: cute.Tensor, @@ -1115,6 +1566,7 @@ def ar_kernel( copy_f32: cute.CopyAtom, store_bf16: cute.CopyAtom, ) -> None: + """The boundary of token ``block // ctas``, one 8-wide vector per thread.""" tidx, _, _ = cute.arch.thread_idx() bid, _, _ = cute.arch.block_idx() grid, _, _ = cute.arch.grid_dim() @@ -1161,6 +1613,19 @@ def ar_kernel( own = Int64(gWs[rank]) for r in cutlass.range_constexpr(nranks): peers[r] = Int64(gWs[r]) + mbar_addr = Int32(0) + if const_expr(self.mbar_exchange): + mbar_addr = self._mbar_setup(tidx) + tot = Int64(m) * Int64(vecs) + idx = Int64(tok) * Int64(vecs) + Int64(vi) + perm = cute.make_rmem_tensor(cute.make_layout((max(self.moe_topk, 1),)), Int32) + weight = cute.make_rmem_tensor(cute.make_layout((max(self.moe_topk, 1),)), Float32) + routing_early: cutlass.Constexpr = self.early_clear and gScale.element_type == BFloat16 + if const_expr(routing_early): + self._load_routing(tok, gPerm, gScale, perm, weight, before_wait=True) + if const_expr(self.early_clear): + cur, dirty, buf_bytes, dirty_stages, stage_sizes = self._mnnvl_state(flags) + self._clear_mnnvl_dirty(own, dirty, buf_bytes, dirty_stages, stage_sizes, idx, tot) if const_expr(self.use_pdl): griddepcontrol_wait() @@ -1172,29 +1637,32 @@ def ar_kernel( # Lamport state below, in one round trip ---- x_in = cute.make_rmem_tensor(cute.make_layout((VEC,)), BFloat16) if const_expr(self.moe_topk > 0): - perm = cute.make_rmem_tensor(cute.make_layout((self.moe_topk,)), Int32) - weight = cute.make_rmem_tensor(cute.make_layout((self.moe_topk,)), Float32) - for k in cutlass.range_constexpr(self.moe_topk): - perm[k] = Int32(gPerm[tok, k]) - weight[k] = gScale[tok, k].to(Float32) + gFc2_v = cute.logical_divide(gFc2, (None, VEC)) + gShared_v = cute.logical_divide(gShared, (None, VEC)) + rows = cute.make_rmem_tensor( + cute.make_layout((self.moe_topk, VEC), stride=(VEC, 1)), BFloat16 + ) + shared = cute.make_rmem_tensor(cute.make_layout((VEC,)), BFloat16) + if const_expr(not routing_early): + self._load_routing(tok, gPerm, gScale, perm, weight, before_wait=False) + if const_expr(self.early_moe_rows): + self._load_moe_rows(tok, vi, perm, gFc2_v, gShared_v, rows, shared, copy_bf16) else: cute.copy(copy_bf16, gX_v[tok, (None, vi)], x_in) # ---- Lamport state of this launch (read before any CTA bumps the arrival counter) ---- if const_expr(self.mnnvl): # Flags (uint32): current, dirty, bytes per buffer, dirty stages, 4 x dirty bytes per - # stage, arrival counter. All of them are read before this CTA arrives: CTA 0 rewrites - # them for the next launch once every CTA has arrived. - cur = _ld_volatile_s32(flags) - dirty = _ld_volatile_s32(flags + Int64(4)) - buf_bytes = Int64(_ld_volatile_s32(flags + Int64(8))) - dirty_stages = _ld_volatile_s32(flags + Int64(12)) - stage_sizes = cute.make_rmem_tensor(cute.make_layout((4,)), Int32) - for s in cutlass.range_constexpr(4): - stage_sizes[s] = _ld_volatile_s32(flags + Int64(16 + 4 * s)) + # stage, arrival counter. All of them are read before this CTA arrives: the CTA counted + # last (CTA 0 without the hand-over warp) rewrites them for the next launch. + if const_expr(not self.early_clear): + cur, dirty, buf_bytes, dirty_stages, stage_sizes = self._mnnvl_state(flags) data_off = Int64(cur) * buf_bytes - # Warp 0 counts the CTA once all its threads have read the flags; the other warps go on. - if tidx < 32: + # Warp 0, or the hand-over warp, counts the CTA once all its threads have read the flags; + # the other warps go on. + if const_expr(self.handover_warp): + _arrive_barrier_1(Int32(self.threads_cta + 32), wait=False) + elif tidx < 32: _arrive_barrier_1(Int32(self.threads_cta), wait=True) if tidx == 0: _red_async_release_add_u32(flags + Int64(32), Int32(1)) @@ -1206,7 +1674,9 @@ def ar_kernel( comm_size = _ld_volatile_s64(layout_addr + Int64(8)) data_off = Int64(flag % 3) * comm_size clear_off = Int64((flag + 2) % 3) * comm_size - if tidx < 32: + if const_expr(self.handover_warp): + _arrive_barrier_1(Int32(self.threads_cta + 32), wait=False) + elif tidx < 32: _arrive_barrier_1(Int32(self.threads_cta), wait=True) if tidx == 0: _atomic_add_s32(flag_addr, Int32(1)) @@ -1216,23 +1686,10 @@ def ar_kernel( griddepcontrol_launch_dependents() # ---- push this rank's slice of the token to every rank ---- - tot = Int64(m) * Int64(vecs) - idx = Int64(tok) * Int64(vecs) + Int64(vi) if const_expr(self.moe_topk > 0): - # MoE finalize: FP32 sum of the routed expert rows and the shared-expert output. All - # rows are loaded before the sum (a skipped row, ``perm < 0``, reads row 0 unused). - gFc2_v = cute.logical_divide(gFc2, (None, VEC)) - gShared_v = cute.logical_divide(gShared, (None, VEC)) - rows = cute.make_rmem_tensor( - cute.make_layout((self.moe_topk, VEC), stride=(VEC, 1)), BFloat16 - ) - for k in cutlass.range_constexpr(self.moe_topk): - p = perm[k] - if p < 0: - p = Int32(0) - cute.copy(copy_bf16, gFc2_v[p, (None, vi)], rows[k, None]) - shared = cute.make_rmem_tensor(cute.make_layout((VEC,)), BFloat16) - cute.copy(copy_bf16, gShared_v[tok, (None, vi)], shared) + # MoE finalize: FP32 sum of the routed expert rows and the shared-expert output. + if const_expr(not self.early_moe_rows): + self._load_moe_rows(tok, vi, perm, gFc2_v, gShared_v, rows, shared, copy_bf16) acc = cute.make_rmem_tensor(cute.make_layout((VEC,)), Float32) for v in cutlass.range_constexpr(VEC): acc[v] = Float32(0.0) @@ -1269,18 +1726,10 @@ def ar_kernel( for r in cutlass.range_constexpr(nranks): _st_v4(peers[r] + data_off + slot, w0, w1, w2, w3) - # ---- clear the slot the previous launch used ---- + # ---- clear the slot the previous launch used (``early_clear``: before the wait) ---- if const_expr(self.mnnvl): - # Stage s of the dirty buffer holds the previous launch's bytes (flags[4 + s]). - for s in cutlass.range_constexpr(4): - if Int32(s) < dirty_stages: - stage_bytes = buf_bytes // Int64(dirty_stages) - stage = own + (Int64(dirty) * Int64(dirty_stages) + Int64(s)) * stage_bytes - clear_vecs = Int64(stage_sizes[s]) // Int64(16) - c = idx - while c < clear_vecs: - _st_v4(stage + c * Int64(16), neg0, neg0, neg0, neg0) - c = c + tot + if const_expr(not self.early_clear): + self._clear_mnnvl_dirty(own, dirty, buf_bytes, dirty_stages, stage_sizes, idx, tot) else: # Its size in bytes (every Lamport kernel on the workspace), as whole 16-byte stores. clear_vecs = (clear_size + Int64(15)) // Int64(16) @@ -1295,7 +1744,12 @@ def ar_kernel( while pending != 0: pending = Int32(0) for r in cutlass.range_constexpr(nranks): - a, b, cc, d = _ld_volatile_v4(own + data_off + (Int64(r) * tot + idx) * Int64(16)) + at = own + data_off + (Int64(r) * tot + idx) * Int64(16) + if const_expr(self.own_slot): + # This rank's slot holds the words just pushed: no wait for their copy. + a, b, cc, d = _ld_volatile_v4_or(at, Int32(rank == r), w0, w1, w2, w3) + else: + a, b, cc, d = _ld_volatile_v4(at) got[r, 0] = a got[r, 1] = b got[r, 2] = cc @@ -1338,28 +1792,20 @@ def ar_kernel( q_addr, sf_addr, gscale, + mbar_addr, ) # ---- hand the triple buffer to the next launch once every CTA has read the flag ---- - if bid == 0: - if tidx == 0: - if const_expr(self.mnnvl): - arrived = _ld_acquire_s32(flags + Int64(32)) - while arrived != grid: + if const_expr(not self.handover_warp): + if bid == 0: + if tidx == 0: + if const_expr(self.mnnvl): arrived = _ld_acquire_s32(flags + Int64(32)) - # This launch's buffer becomes the dirty one (one stage, all ranks' slices). - _st_s32(flags, (cur + 1) % 3) - _st_s32(flags + Int64(4), cur) - _st_s32(flags + Int64(12), Int32(1)) - _st_s32(flags + Int64(16), Int32(m) * Int32(self.hidden * nranks * 2)) - _st_s32(flags + Int64(20), Int32(0)) - _st_s32(flags + Int64(24), Int32(0)) - _st_s32(flags + Int64(28), Int32(0)) - _st_s32(flags + Int64(32), Int32(0)) - else: - arrived = _ld_volatile_s32(flag_addr) - while arrived != grid: + while arrived != grid: + arrived = _ld_acquire_s32(flags + Int64(32)) + self._advance_state(cur, m, flags, flags) + else: arrived = _ld_volatile_s32(flag_addr) - _st_s32(flag_addr + Int64(8), (flag + 1) % 3) - _st_s64(layout_addr, Int64(m) * Int64(self.hidden * nranks * 2)) - _st_s32(flag_addr, Int32(0)) + while arrived != grid: + arrived = _ld_volatile_s32(flag_addr) + self._advance_state(flag, m, flag_addr, layout_addr) diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv41.py b/tensorrt_llm/_torch/models/modeling_deepseekv41.py index 68ded6123132..ef1c89e4a814 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv41.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv41.py @@ -68,6 +68,7 @@ mhc_lamport_allreduce, mhc_post_collapse_norm, mhc_split_enabled, + ob_boundary_enabled, ) from ..modules.multi_stream_utils import do_multi_stream from ..moe.fused_moe.impl_contract import MOE_DECODE_OPTIONS, MoEDecodeOptions @@ -1180,6 +1181,16 @@ def _mhc_fused_allreduce(self): self._mhc_lamport_resolved = True return self._mhc_lamport + def _ob_weights(self) -> tuple: + """The MXFP8 o_b projection's weight and linear scales, as the deferred projection reads them.""" + from ..modules.low_m_gemm import mxfp8_linear_scale + + linear = self.self_attn.o_b_proj + if linear.weight.dtype != torch.float8_e4m3fn: + return () + scale = mxfp8_linear_scale(linear) + return (linear.weight,) if scale is None else (linear.weight, scale) + def _moe_supports_unfinalized(self) -> bool: """Whether the routed experts can hand back their rows before the top-k combine.""" experts = self.mlp.experts @@ -1299,6 +1310,8 @@ def forward( ffn_split = self._mhc_split(residual.shape[0], ffn=True) # Split boundaries can also all-reduce the sublayer output in their own launch. lamport = self._mhc_fused_allreduce() if split else None + # The attention may hand back its o_b input for the FFN boundary to project in its launch. + defer_ob = lamport is not None and ffn_split and ob_boundary_enabled(residual.shape[0]) attn_coeff_event = None # The entry boundary (split or single-pass) also emits the MXFP8 input of the fused # query/SWA projection. @@ -1324,6 +1337,10 @@ def forward( events=self._mhc_attn_events, allreduce=entry_allreduce, mxfp8=mxfp8, + # The split FFN boundary reads these coefficients after its dependency wait. + coefficients_release_after_wait=ffn_split, + # Its o_b CTAs then read their weights from L2. + l2_prefetch=self._ob_weights() if defer_ob and entry_allreduce is not None else (), ) residual, attn_pre, post_mix, comb_mix, layer_input, attn_coeff_event = boundary[:6] if mxfp8: @@ -1361,19 +1378,23 @@ def forward( residual.reshape(residual.shape[0], -1), getattr(attn_metadata, "csa2_replay_query_rows", None), ) - x_attn = self.self_attn( - position_ids=position_ids, - hidden_states=layer_input, - attn_metadata=attn_metadata, - hidden_states_mxfp8=layer_input_mxfp8, - # The split boundary's critical half prefetches its residual before its PDL wait, - # so the all-reduce may let it launch at its start. - all_reduce_params=AllReduceParams( - enable_allreduce=not self.disable_attn_allreduce and lamport is None, - trigger_completion_at_end=not split, - ), - **kwargs, - ) + self.self_attn.defer_ob_to_boundary = defer_ob + try: + x_attn = self.self_attn( + position_ids=position_ids, + hidden_states=layer_input, + attn_metadata=attn_metadata, + hidden_states_mxfp8=layer_input_mxfp8, + # The split FFN boundary loads its constant inputs, and after a split entry also + # its residual, before its PDL wait, so the all-reduce may let it launch at its start. + all_reduce_params=AllReduceParams( + enable_allreduce=not self.disable_attn_allreduce and lamport is None, + trigger_completion_at_end=not ffn_split, + ), + **kwargs, + ) + finally: + self.self_attn.defer_ob_to_boundary = False if after_attention_callback is not None: after_attention_callback() if capture_done is not None: @@ -1785,6 +1806,9 @@ def forward(self, attn_metadata, *args, **kwargs): self._engram_overlap_decision = self._use_engram_context_overlap(attn_metadata) try: output = super().forward(attn_metadata, *args, **kwargs) + join_swa_refresh = getattr(attn_metadata, "join_swa_refresh_fork", None) + if join_swa_refresh is not None: + join_swa_refresh() finally: self._prologue_start = None self._engram_overlap_decision = None diff --git a/tensorrt_llm/_torch/modules/low_m_gemm.py b/tensorrt_llm/_torch/modules/low_m_gemm.py index 6c820dd8845f..76f3ec8e037b 100644 --- a/tensorrt_llm/_torch/modules/low_m_gemm.py +++ b/tensorrt_llm/_torch/modules/low_m_gemm.py @@ -1067,6 +1067,7 @@ def _compiled_mma( bf16_weights: bool = False, out_dtype: torch.dtype = torch.bfloat16, k_warps: int = 0, + smem_carveout: int = 0, a_rope_head_dim: int = 0, a_rms_eps: Optional[float] = None, n_tiles: int = 1, @@ -1128,7 +1129,7 @@ def _compiled_mma( cos_sin = torch.empty((max(rope_positions, 1), 2, 32), dtype=torch.float32, device=dev) gamma = torch.empty((1, k if a_rms_eps is not None else 8), dtype=torch.bfloat16, device=dev) stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - return cute.compile( + compiled = cute.compile( kernel, graph_safe_from_dlpack(a), graph_safe_from_dlpack(b), @@ -1140,6 +1141,11 @@ def _compiled_mma( graph_safe_from_dlpack(gamma), stream, ) + if smem_carveout: + from ..attention.backends.sparse.csa2.direct_decode import _set_smem_carveout + + _set_smem_carveout(compiled, smem_carveout) + return compiled # Row counts of the MMA kernels outside decode CUDA graphs: each up to 8 rows, as decode batches @@ -1387,6 +1393,7 @@ def mxfp8_lowm_linear( split_n: int = 0, rope: Optional[Tuple[torch.Tensor, torch.Tensor, int]] = None, scale_swizzled: bool = False, + smem_carveout: int = 0, rms_norm: Optional[Tuple[torch.Tensor, float]] = None, n_tiles: int = 1, swiglu_out_limit: Optional[float] = None, @@ -1400,6 +1407,8 @@ def mxfp8_lowm_linear( tensors ``(y[:, :split_n], y[:, split_n:])`` written directly by the kernel. ``rope``: (positions [M] int, cos/sin [P, 2, 32] fp32, head_dim) applies interleaved RoPE to the last 64 columns of every head of the output. + ``smem_carveout``: the kernel's preferred shared-memory carveout in percent (0: the driver's + choice); the result is unchanged. ``rms_norm``: (gamma [K] bf16, eps) projects the RMS-normalized rows of x, ``rms_norm(x) * gamma``, instead of x. ``n_tiles`` (N a multiple of ``8 * n_tiles``): 8-column tiles per CTA, the same result with fewer CTAs. @@ -1429,6 +1438,7 @@ def mxfp8_lowm_linear( 0 if rope is None else rope[2], 0 if rope is None else rope[1].shape[0], scale_swizzled=scale_swizzled, + smem_carveout=smem_carveout, a_rms_eps=None if rms_norm is None else float(rms_norm[1]), n_tiles=n_tiles, bf16_dequant=bf16_dequant, diff --git a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py index 2ab7aab2d30d..77520f867625 100644 --- a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py +++ b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py @@ -817,6 +817,7 @@ def split_hc_lagged( nvfp4_scale: Optional[torch.Tensor] = None, residual_after_wait: bool = False, coefficients_release_after_wait: bool = False, + l2_prefetch: tuple = (), ) -> tuple: """Lagged boundary as a critical half and a coefficient half (decode-sized M). @@ -835,7 +836,8 @@ def split_hc_lagged( dependency wait (a split boundary), so their launch may release it from its own wait on. With ``allreduce`` (a :class:`~.mhc_lowm.LamportAllReduce`), ``x_prev`` is this rank's - tensor-parallel partial sum and the critical half all-reduces it in the same launch. + tensor-parallel partial sum and the critical half all-reduces it in the same launch; it + also reads the ``l2_prefetch`` tensors into L2 (see ``mhc_allreduce_post_collapse_norm``). Returns ``(residual, pre, post, comb, layer_input, coeff_event)``; ``residual`` is ``residual_prev`` itself when ``x_prev`` is None. ``mxfp8`` appends ``layer_input``'s @@ -878,6 +880,7 @@ def split_hc_lagged( allreduce, mxfp8=mxfp8, nvfp4_scale=nvfp4_scale, + l2_prefetch=l2_prefetch, ) else: residual, layer_input, *quantized = mhc_post_collapse_norm( diff --git a/tensorrt_llm/_torch/modules/mhc/mhc_lowm.py b/tensorrt_llm/_torch/modules/mhc/mhc_lowm.py index 244d77e135c4..2ce4fc8dd89c 100644 --- a/tensorrt_llm/_torch/modules/mhc/mhc_lowm.py +++ b/tensorrt_llm/_torch/modules/mhc/mhc_lowm.py @@ -282,6 +282,125 @@ def finalize(self) -> torch.Tensor: return (routed + self.shared.float()).to(self.shared.dtype) +# Rows up to which the attention's o_b runs inside the all-reduce boundary: one MMA row tile. +_OB_BOUNDARY_MAX_ROWS = 16 + + +@dataclass +class ObUnprojected: + """A TP rank's attention output before its row-parallel small-M MXFP8 ``o_b`` projection; the + all-reduce boundary that follows projects, all-reduces and finishes it in one launch.""" + + o_lat: torch.Tensor + weight: torch.Tensor + scale_lin: torch.Tensor + k_warps: int + + @property + def shape(self): + return torch.Size((self.o_lat.shape[0], self.weight.shape[0])) + + +def ob_boundary_enabled(rows: int) -> bool: + """Whether a decode boundary of ``rows`` rows takes the attention's ``o_b`` into its launch.""" + return 0 < rows <= _OB_BOUNDARY_MAX_ROWS and is_blackwell_datacenter() + + +def defer_ob_projection( + o_lat: torch.Tensor, weight: torch.Tensor, scale_lin: torch.Tensor | None +) -> ObUnprojected | None: + """``mxfp8_lowm_linear(o_lat, weight, scale)`` deferred to the boundary, or None where the fused + launch cannot reproduce it.""" + from ..low_m_gemm import default_k_warps + + if scale_lin is None or o_lat.dtype != torch.bfloat16 or weight.dtype != torch.float8_e4m3fn: + return None + rows, k = o_lat.shape + n = weight.shape[0] + k_warps = default_k_warps(k) + if ( + not 0 < rows <= _OB_BOUNDARY_MAX_ROWS + or k_warps not in (2, 4) + or not _ob_tiles(n, k, k_warps) + ): + return None + return ObUnprojected(o_lat.contiguous(), weight, scale_lin, k_warps) + + +@functools.lru_cache(maxsize=None) +def _ob_tiles(hidden: int, k: int, k_warps: int) -> bool: + from ...kernels.blackwell.mhc.mhc_ob_boundary import ob_boundary_tiles + + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + return ob_boundary_tiles(hidden, k, k_warps, props.shared_memory_per_block_optin) + + +@functools.lru_cache(maxsize=None) +def _compiled_ob_boundary( + m: int, + hidden: int, + k: int, + k_warps: int, + norm_eps: float, + nranks: int, + use_pdl: bool, + mnnvl: bool, + quant: str | None, + mbar_exchange: bool, + bf16_dequant: bool, +): + import cuda.bindings.driver as cuda + import cutlass + import cutlass.cute as cute + + from ...kernels.blackwell.mhc.mhc_ob_boundary import ObAllReduceBoundaryKernel + + kernel = ObAllReduceBoundaryKernel( + hidden=hidden, + k_extent=k, + num_rows=m, + k_warps=k_warps, + norm_eps=norm_eps, + nranks=nranks, + use_pdl=use_pdl, + mnnvl=mnnvl, + quant=quant, + mbar_exchange=mbar_exchange, + bf16_dequant=bf16_dequant, + ) + dev = torch.device("cuda") + a = torch.empty((m, k), dtype=torch.bfloat16, device=dev) + b = torch.empty((hidden, k), dtype=torch.uint8, device=dev) + s = torch.empty((hidden, k // 32), dtype=torch.uint8, device=dev) + res = torch.empty((m, 4 * hidden), dtype=torch.bfloat16, device=dev) + post = torch.empty((m, 4), dtype=torch.float32, device=dev) + comb = torch.empty((m, 16), dtype=torch.float32, device=dev) + pre = torch.empty((m, 4), dtype=torch.float32, device=dev) + w = torch.empty((1, hidden), dtype=torch.bfloat16, device=dev) + out = torch.empty((m, hidden), dtype=torch.bfloat16, device=dev) + table = torch.empty((3 if mnnvl else nranks + 2,), dtype=torch.int64, device=dev) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + return cute.compile( + kernel, + graph_safe_from_dlpack(a), + graph_safe_from_dlpack(b), + graph_safe_from_dlpack(s), + graph_safe_from_dlpack(res), + graph_safe_from_dlpack(post), + graph_safe_from_dlpack(comb), + graph_safe_from_dlpack(pre), + graph_safe_from_dlpack(w), + graph_safe_from_dlpack(res), + graph_safe_from_dlpack(out), + graph_safe_from_dlpack(table), + cutlass.Int32(0), + cutlass.Int64(0), + cutlass.Int64(0), + cutlass.Int64(0), + stream, + ) + + @functools.lru_cache(maxsize=None) def _placeholder_moe_operands(hidden: int, device_index: int): dev = torch.device("cuda", device_index) @@ -293,6 +412,32 @@ def _placeholder_moe_operands(hidden: int, device_index: int): ) +_EARLY_CLEAR_LOGGED: set = set() + + +def _early_clear(x_partial, allreduce, use_pdl: bool, m: int) -> bool: + """Whether the MNNVL MoE-finalize boundary clears before its wait: only right behind a captured + CuTe FC2, which starts it after its own wait.""" + if not ( + use_pdl + and allreduce.mnnvl + and isinstance(x_partial, MoeUnfinalized) + and is_blackwell_datacenter() + ): + return False + from ...moe.fused_moe.trtllm_gen import decode_moe + + ok, why = decode_moe.precedes(x_partial.fc2) + if (m, why) not in _EARLY_CLEAR_LOGGED: + _EARLY_CLEAR_LOGGED.add((m, why)) + from tensorrt_llm.logger import logger + + logger.debug( + f"mHC MoE all-reduce boundary, {m} rows: early clear {'on' if ok else 'off'} ({why})" + ) + return ok + + @functools.lru_cache(maxsize=None) def _compiled_allreduce_boundary( m: int, @@ -304,6 +449,12 @@ def _compiled_allreduce_boundary( ctas_per_token: int = 1, mnnvl: bool = False, quant: str | None = None, + mbar_exchange: bool = False, + early_moe_rows: bool = False, + early_clear: bool = False, + own_slot: bool = False, + handover_warp: bool = False, + l2_prefetch: bool = False, ): """``moe`` = (fc2 rows, top_k, scale dtype) for the MoE-finalize input, None otherwise.""" import cuda.bindings.driver as cuda @@ -321,6 +472,12 @@ def _compiled_allreduce_boundary( ctas_per_token=ctas_per_token, mnnvl=mnnvl, quant=quant, + mbar_exchange=mbar_exchange, + early_moe_rows=early_moe_rows, + early_clear=early_clear, + own_slot=own_slot, + handover_warp=handover_warp, + l2_prefetch=l2_prefetch, ) dev = torch.device("cuda") x = torch.empty((m, hidden), dtype=torch.bfloat16, device=dev) @@ -358,6 +515,10 @@ def _compiled_allreduce_boundary( cutlass.Int64(0), cutlass.Int64(0), cutlass.Int64(0), + cutlass.Int64(0), + cutlass.Int64(0), + cutlass.Int64(0), + cutlass.Int64(0), stream, ) @@ -373,6 +534,7 @@ def mhc_allreduce_post_collapse_norm( allreduce: "LamportAllReduce | MnnvlLamportAllReduce", mxfp8: bool = False, nvfp4_scale: torch.Tensor | None = None, + l2_prefetch: tuple[torch.Tensor, ...] = (), ) -> tuple[torch.Tensor, ...]: """:func:`mhc_post_collapse_norm` of the TP sum of ``x_partial`` (one-shot all-reduce in-kernel). @@ -380,7 +542,9 @@ def mhc_allreduce_post_collapse_norm( so the outputs equal ``mhc_post_collapse_norm(allreduce(x_partial), ...)`` bit for bit. A :class:`MoeUnfinalized` partial is finalized in the same launch (``x_partial.finalize()``, up to the FP32 summation order). ``mxfp8`` / ``nvfp4_scale`` add a quantized copy of the - normalized output as :func:`mhc_post_collapse_norm` does. + normalized output as :func:`mhc_post_collapse_norm` does. Up to two contiguous + ``l2_prefetch`` tensors (sizes multiples of 16 bytes) are read into L2 by the launch, for a + later kernel; an ``ObUnprojected`` partial ignores them. """ import cuda.bindings.driver as cuda import cutlass @@ -388,6 +552,19 @@ def mhc_allreduce_post_collapse_norm( m, mult, hidden = residual_prev.shape assert mult == 4 and residual_prev.dtype == torch.bfloat16 res_flat = residual_prev.reshape(m, mult * hidden).contiguous() + if isinstance(x_partial, ObUnprojected): + return _ob_allreduce_post_collapse_norm( + x_partial, + res_flat, + post_mix_prev, + comb_mix_prev, + pre_mix, + norm_weight, + norm_eps, + allreduce, + mxfp8, + nvfp4_scale, + ) moe_key = None if isinstance(x_partial, MoeUnfinalized): fc2 = x_partial.fc2 @@ -418,18 +595,29 @@ def mhc_allreduce_post_collapse_norm( ] # Unfinalized MoE uses shared as x too; keep the alias to avoid a second copy. x = shared.view(m, hidden) if moe_key is not None else (x.clone() if x.data_ptr() % 16 else x) + use_pdl = get_env_enable_pdl() fn = _compiled_allreduce_boundary( m, hidden, float(norm_eps), allreduce.nranks, - get_env_enable_pdl(), + use_pdl, moe_key, # A cluster of 4 CTAs per token for the smallest batches, when the token's warps # (8 channels per thread) split evenly over them. 4 if m <= 12 and hidden % (4 * 32 * 8) == 0 else 1, mnnvl=allreduce.mnnvl, quant=quant, + mbar_exchange=is_blackwell_datacenter(), + early_moe_rows=is_blackwell_datacenter(), + early_clear=_early_clear(x_partial, allreduce, use_pdl, m), + own_slot=is_blackwell_datacenter(), + handover_warp=is_blackwell_datacenter(), + l2_prefetch=bool(l2_prefetch), + ) + regions = [(t.data_ptr(), t.numel() * t.element_size()) for t in l2_prefetch] + [(0, 0)] * 2 + assert len(l2_prefetch) <= 2 and all( + t.is_contiguous() and n % 16 == 0 for t, (_, n) in zip(l2_prefetch, regions) ) stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) fn( @@ -450,6 +638,76 @@ def mhc_allreduce_post_collapse_norm( cutlass.Int64(quantized[0].data_ptr() if quant else 0), cutlass.Int64(quantized[1].data_ptr() if quant else 0), cutlass.Int64(scale_addr), + cutlass.Int64(regions[0][0]), + cutlass.Int64(regions[0][1]), + cutlass.Int64(regions[1][0]), + cutlass.Int64(regions[1][1]), + stream, + ) + if quant: + return res_out.view(m, mult, hidden), out, quantized + return res_out.view(m, mult, hidden), out + + +def _ob_allreduce_post_collapse_norm( + x_partial: ObUnprojected, + res_flat: torch.Tensor, + post_mix_prev: torch.Tensor, + comb_mix_prev: torch.Tensor, + pre_mix: torch.Tensor, + norm_weight: torch.Tensor, + norm_eps: float, + allreduce: "LamportAllReduce | MnnvlLamportAllReduce", + mxfp8: bool, + nvfp4_scale: torch.Tensor | None, +) -> tuple[torch.Tensor, ...]: + """``mhc_allreduce_post_collapse_norm`` of ``o_b(o_lat)``, the projection in the same launch.""" + import cuda.bindings.driver as cuda + import cutlass + + from ..low_m_gemm import lowm_mma_options + + m, k = x_partial.o_lat.shape + hidden = x_partial.weight.shape[0] + mult = res_flat.shape[1] // hidden + pre = pre_mix.reshape(m, mult).to(torch.float32).contiguous() + post = post_mix_prev.reshape(m, mult).to(torch.float32).contiguous() + comb = comb_mix_prev.reshape(m, mult * mult).to(torch.float32).contiguous() + weight = norm_weight.to(torch.bfloat16).contiguous().view(1, hidden) + out = torch.empty((m, hidden), dtype=torch.bfloat16, device=res_flat.device) + res_out = torch.empty_like(res_flat) + quant, quantized, scale_addr = _quantized_outputs(m, hidden, out.device, mxfp8, nvfp4_scale) + table = allreduce.prepare(allreduce.nranks * m * hidden * res_flat.element_size()) + fn = _compiled_ob_boundary( + m, + hidden, + k, + x_partial.k_warps, + float(norm_eps), + allreduce.nranks, + get_env_enable_pdl(), + allreduce.mnnvl, + quant, + is_blackwell_datacenter(), + lowm_mma_options()[0], + ) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + fn( + graph_safe_from_dlpack(x_partial.o_lat), + graph_safe_from_dlpack(x_partial.weight.view(torch.uint8)), + graph_safe_from_dlpack(x_partial.scale_lin), + graph_safe_from_dlpack(res_flat), + graph_safe_from_dlpack(post), + graph_safe_from_dlpack(comb), + graph_safe_from_dlpack(pre), + graph_safe_from_dlpack(weight), + graph_safe_from_dlpack(res_out), + graph_safe_from_dlpack(out), + graph_safe_from_dlpack(table), + cutlass.Int32(allreduce.rank), + cutlass.Int64(quantized[0].data_ptr() if quant else 0), + cutlass.Int64(quantized[1].data_ptr() if quant else 0), + cutlass.Int64(scale_addr), stream, ) if quant: diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 6280f666a94a..81be04809d13 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -2791,6 +2791,10 @@ "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu/test_allreduce.py::test_allreduce_lamport_mixed_dtype]": 43.06712676056338, "unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_allreduce_boundary_matches_allreduce_then_boundary": 41.647438848920864, "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_allreduce_boundary_matches_allreduce_then_boundary]": 41.647438848920864, + "unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_moe_boundary_behind_lowm_fc2": 41.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_moe_boundary_behind_lowm_fc2]": 41.0, + "unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_ob_boundary_matches_ob_then_boundary": 59.0, + "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_ob_boundary_matches_ob_then_boundary]": 59.0, "unittest/_torch/multi_gpu/test_allreduce.py::test_minimax_allreduce_rms_odd_tokens": 42.062401408450704, "test_unittests.py::test_unittests_v2[unittest/_torch/multi_gpu/test_allreduce.py::test_minimax_allreduce_rms_odd_tokens]": 42.062401408450704, "unittest/_torch/multi_gpu/test_allreduce.py::test_nccl_symmetric_unregistered_small_input": 35.3064609929078, diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index b525db7462b4..65c204443fc2 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -58,6 +58,9 @@ l0_gb200_multi_gpus: - unittest/_torch/multi_gpu/test_peer_buffers.py - unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_allreduce_boundary_matches_allreduce_then_boundary - unittest/_torch/multi_gpu/test_mnnvl_allreduce.py::test_mnnvl_rms_norm_without_residual + # mHC all-reduce boundaries that take the attention O-B, or wait behind the CuTe DSL FC2. + - unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_ob_boundary_matches_ob_then_boundary + - unittest/_torch/multi_gpu/test_allreduce.py::test_mhc_moe_boundary_behind_lowm_fc2 # ---- curated pre-merge smoke set: diversify moe_tactic across cutlass/trtllm_gen/cutedsl.grouped_gemm ---- - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_tactic=cutlass-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_tactic=trtllm_gen-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] diff --git a/tests/unittest/_torch/attention/sparse/csa2/_utils.py b/tests/unittest/_torch/attention/sparse/csa2/_utils.py index 1ffce580968b..c22a38163bb8 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/_utils.py +++ b/tests/unittest/_torch/attention/sparse/csa2/_utils.py @@ -43,7 +43,16 @@ def write_global(owner, slots, values, keys, main_transform=None, index_transfor store_rows(main[owner], slots, values, "main", main_transform) store_index_rows(index[owner], slots, keys, index_transform) - def write_layer_rows(layer, slots, values, transform=None, *, global_rows=None, index_q=None): + def write_layer_rows( + layer, + slots, + values, + transform=None, + *, + global_rows=None, + index_q=None, + release_after_wait=False, + ): store_rows(swa[layer], slots, values, "swa", transform) if global_rows is not None: write_global(*global_rows) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_direct_decode.py b/tests/unittest/_torch/attention/sparse/csa2/test_direct_decode.py index 77f89ae48bdd..fd5a7516f754 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_direct_decode.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_direct_decode.py @@ -6,6 +6,7 @@ import pytest import torch +from tensorrt_llm._torch.attention.backends.sparse.csa2 import direct_decode from tensorrt_llm._torch.attention.backends.sparse.csa2.direct_decode import direct_sparse_decode from tensorrt_llm._torch.attention.backends.sparse.csa2.quantization import pack_rows, unpack_rows @@ -152,6 +153,65 @@ def test_matches_fp32_reference(rows, heads, main_width): assert torch.equal(_bits(_run(case, query_rope=(positions, cos_sin))), _bits(expected)) +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ((10, 0), (10, 3)), + reason="SM100/SM103 small-M O-A", +) +@pytest.mark.parametrize("rows,heads", [(6, 8)]) +@torch.inference_mode() +def test_output_rotated_back_gives_the_o_a_latent_bitwise(rows, heads): + """The output rotated back in the launch gives the small-M O-A without inverse RoPE the latent + of the O-A that rotates on load, bit for bit, eagerly and in graph replays.""" + from tensorrt_llm._torch.modules.low_m_gemm import fp8_grouped_lowm, lowm_tiles + + case = _case(rows, heads, 128, 512, mapped=True, seed=rows + heads) + q = case[0] + g = torch.Generator(device="cuda").manual_seed(rows * heads) + positions = torch.randint(0, 4096, (rows,), device="cuda", generator=g, dtype=torch.int32) + angle = torch.rand(4096, 32, device="cuda", generator=g) * 6.2831853 + cos_sin = torch.cat((angle.cos(), angle.sin()), 1).view(4096, 2, 32) + groups, rank, depth = heads // 8, 1024, 8 * 512 + # Latent row r of a group passes one channel of head (r % 512) // 64: RoPE ones below 512. + r = torch.arange(rank, device="cuda") + head = (r % 512) // 64 + column = torch.where(r < 512, head * 512 + 448 + r % 64, head * 512 + (r % 64) * 7) + weight = torch.zeros(groups * rank, depth, device="cuda") + weight[torch.arange(groups * rank, device="cuda"), column.repeat(groups)] = 1.0 + weight = weight.to(torch.float8_e4m3fn) + scale = torch.ones(groups * rank // 128, depth // 128, device="cuda") + tiles = lowm_tiles(groups * rank, rows, rank) + + def project(rotated_back): + out = _run(case, query_rope=(positions, cos_sin), output_rope=rotated_back) + latent = fp8_grouped_lowm( + out.view(rows, -1), + weight, + scale, + rank, + inverse_rope=None if rotated_back else (positions, cos_sin, 512), + n_tiles=tiles, + ) + return out, latent + + def check(rotated, plain): + assert torch.equal(_bits(rotated[0][..., :448]), _bits(plain[0][..., :448])) + assert torch.equal(_bits(rotated[1]), _bits(plain[1])) + + check(project(True), project(False)) + graphs, captured = [], [] + for rotated_back in (True, False): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured.append(project(rotated_back)) + graphs.append(graph) + for trial in range(2): + q.copy_((torch.randn_like(q, dtype=torch.float32) * (trial + 1)).to(torch.bfloat16)) + positions.random_(0, 4096, generator=g) + for graph in graphs: + graph.replay() + check(*captured) + + @pytest.mark.parametrize("rows", [12]) @pytest.mark.parametrize("heads", [8]) @torch.inference_mode() @@ -305,3 +365,91 @@ def test_exported_slots_replace_the_mapping_bitwise(rows, heads): assert torch.equal(_bits(_run(case, slots_out=slots)), out) assert torch.equal(slots, _reference_slots(mi, mapping)) assert torch.equal(_bits(_run((q, swa, si, main, slots, sink, None))), out) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ((10, 0), (10, 3)), + reason="SM100/SM103 keys-as-M attention", +) +@torch.inference_mode() +def test_exported_main_rows_replace_the_decode_bitwise(): + """An index source storing its decoded main rows is unchanged, and a Reuse launch loading them + equals the one decoding them from the exported slots, eagerly and in graph replays.""" + rows = 6 + case = _case(rows, 8, 128, 512, mapped=True, seed=46) + q, swa, si, main, mi, sink, mapping = case + image = direct_decode.main_rows_bytes(8, 128, 512, q.device) + slots = torch.empty(rows, 512, dtype=torch.int64, device="cuda") + main_rows = torch.full((rows, image), 0xFF, dtype=torch.uint8, device="cuda") + outputs = [torch.empty_like(q) for _ in range(4)] + reuse = (q, swa, si, main, slots, sink, None) + + def forward(): + _run(case, output=outputs[0]) + _run(case, output=outputs[1], slots_out=slots, rows_out=main_rows) + # main_ready: the launch before a Reuse decode starts only once the source completed. + slots.add_(0) + _run(reuse, output=outputs[2], main_ready=True) + _run(reuse, output=outputs[3], main_ready=True, rows_in=main_rows) + + def check(): + torch.cuda.synchronize() + assert torch.equal(_bits(outputs[1]), _bits(outputs[0])) + assert torch.equal(_bits(outputs[3]), _bits(outputs[2])) + _assert_close_to_fp32(outputs[3], _reference(case)) + + forward() + check() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + forward() + for trial in range(2): + q.copy_((torch.randn_like(q, dtype=torch.float32) * (trial + 1) * 0.5).to(torch.bfloat16)) + mi.copy_(mi.flip(1)) + graph.replay() + check() + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ((10, 0), (10, 3)), + reason="SM100/SM103 keys-as-M attention", +) +@pytest.mark.parametrize("selection", ["mapped"]) +@torch.inference_mode() +def test_keys_as_m_attention_matches_the_two_part_launch(monkeypatch, selection): + """At 8 heads per rank the keys-as-M attention matches the FP32 reference as the two-part launch + does, eagerly and in graph replays; a row without valid keys outputs zeros.""" + main_width = 0 if selection == "swa" else 512 + case = _case(6, 8, 128, main_width, mapped=selection == "mapped", seed=76) + q, swa, si, main, mi, sink, mapping = case + si[-1] = -1 + if mi is not None: + mi[-1] = -1 + with monkeypatch.context() as patch: + patch.setattr(direct_decode, "_v2_engaged", lambda *args: False) + two_part = _run(case).clone() + output = torch.empty_like(q) + _run(case, output=output) + _assert_close_to_fp32(output, _reference(case)) + _assert_close_to_fp32(output, two_part) + assert torch.count_nonzero(output[-1]) == 0 + # A Reuse layer's launch (main rows promised before the dependent wait) gives the same bits. + ready = torch.empty_like(q) + if main_width: + _run(case, output=ready, main_ready=True) + assert torch.equal(_bits(ready), _bits(output)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _run(case, output=output) + if main_width: + _run(case, output=ready, main_ready=True) + for trial in range(2): + q.copy_((torch.randn_like(q, dtype=torch.float32) * (trial + 1) * 0.5).to(torch.bfloat16)) + si.copy_(si.roll(1, 0)) + if mi is not None: + mi.copy_(mi.flip(1)) + graph.replay() + torch.cuda.synchronize() + _assert_close_to_fp32(output, _reference(case)) + if main_width: + assert torch.equal(_bits(ready), _bits(output)) diff --git a/tests/unittest/_torch/attention/sparse/csa2/test_module.py b/tests/unittest/_torch/attention/sparse/csa2/test_module.py index c6fb773b001e..9a3b56dbc453 100644 --- a/tests/unittest/_torch/attention/sparse/csa2/test_module.py +++ b/tests/unittest/_torch/attention/sparse/csa2/test_module.py @@ -88,7 +88,16 @@ def write_global(layer, slots, main, index, main_transform=None, index_transform torch.testing.assert_close(index, hidden[:, :128], atol=0, rtol=0) published.append(("global", layer)) - def write_layer_rows(layer, slots, values, transform=None, *, global_rows=None, index_q=None): + def write_layer_rows( + layer, + slots, + values, + transform=None, + *, + global_rows=None, + index_q=None, + release_after_wait=False, + ): write_swa(layer, slots, values, transform) if global_rows is not None: write_global(*global_rows) @@ -144,6 +153,7 @@ def indexer(state, start, end): native_fp8=False, fused_decode=False, gen_fmha_serves=lambda *args: False, + direct_decode_applies=lambda *args: False, layer_idx=0, num_heads=1, indexer=indexer, @@ -151,8 +161,11 @@ def indexer(state, start, end): backend._native_slot_mapping = lambda logical: CSA2TrtllmAttention._native_slot_mapping( backend, logical ) - backend._resolve_sparse_inputs = lambda count, args: ( - CSA2TrtllmAttention._resolve_sparse_inputs(backend, count, args) + backend.publish_layer_rows = lambda state, **kwargs: CSA2TrtllmAttention.publish_layer_rows( + backend, state, **kwargs + ) + backend._resolve_sparse_inputs = lambda count, args, **kwargs: ( + CSA2TrtllmAttention._resolve_sparse_inputs(backend, count, args, **kwargs) ) def compute(query, k, v, metadata, *, forward_args): @@ -215,7 +228,9 @@ def flash_compute(phase, query, metadata, args): global_hidden, global_hidden[:, :128], ) - model._project_output = lambda output, position, inverse_rope_applied: output.flatten(1) + model._project_output = lambda output, position, inverse_rope_applied, rotated_back: ( + output.flatten(1) + ) actual = model(hidden, positions, source) expected = hidden + (row_ids[:, None].remainder(7) + 1).to(hidden.dtype) torch.testing.assert_close(actual, expected, atol=0, rtol=0) @@ -742,16 +757,21 @@ def gemm(*args, **kwargs): @pytest.mark.parametrize("lowm_qkv", [False, True]) @pytest.mark.parametrize("rows", [6]) @torch.inference_mode() -def test_decode_graph_query_projections_match_unfused(rows, lowm_qkv): +def test_decode_graph_query_projections_match_unfused(monkeypatch, rows, lowm_qkv): """Decode-graph query/SWA, Q and index-Q projections (index Q with its RoPE) on the small-M kernel, with and without the fused Q-A and KV projection on it, match the plain path at the model's shapes (hidden 5120, 8 heads, 32 index heads).""" + from tensorrt_llm._torch.attention.backends.sparse.csa2 import module as csa2_module from tensorrt_llm._torch.autotuner import AutoTuner from tensorrt_llm._torch.modules.linear import ( flashinfer_mxfp8_autotune, flashinfer_mxfp8_decode_graph_capture, ) + # The BF16 index query, which the cache write packs. + monkeypatch.setattr( + csa2_module.DeepseekV41Attention, "_packs_index_q", lambda self, rows, positions: False + ) torch.manual_seed(946) model, _ = _native_model(q_lora_rank=1280, hidden_size=5120) model._lowm_qkv_projection = lowm_qkv @@ -1084,6 +1104,135 @@ def delayed_descriptors(self, *args): torch.testing.assert_close(parallel, serial, atol=0, rtol=0, msg=f"{cached} cached tokens") +@skip_pre_blackwell +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@torch.inference_mode() +def test_index_query_pack_publishes_rows_beside_index_chain(monkeypatch, module_cache): + """A direct-decode step whose index-query projection packs the queries (cache rows published on a + side stream) leaves the cache bytes, selections and output of the cache write's packing.""" + from tensorrt_llm._torch.attention.backends.sparse.csa2 import cache_manager, quantization + from tensorrt_llm._torch.attention.backends.sparse.csa2 import module as csa2_module + from tensorrt_llm._torch.attention.backends.sparse.csa2.metadata import CSA2TrtllmMetadata + from tensorrt_llm._torch.metadata import KVCacheParams + from tensorrt_llm._torch.modules import low_m_gemm + from tensorrt_llm._torch.modules.linear import ( + flashinfer_mxfp8_autotune, + flashinfer_mxfp8_decode_graph_capture, + ) + + if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)): + pytest.skip("The packing projection runs on SM100/SM103") + torch.manual_seed(951) + # The kernel against the cache write's packing of the BF16 projection. + kernel_model, _ = _native_model(q_lora_rank=1280) + linear = kernel_model.index_wq_b + cos_sin = kernel_model.rotary_emb.rotary_cos_sin.view(-1, 2, 32) + for rows in (1, 6, 12, 16): + qr = torch.randn(rows, 1280, device="cuda", dtype=torch.bfloat16) + for row, scale in zip(range(1, rows), (0.0, 1e-30, 1e30, float("nan"), float("inf"))): + qr[row] *= scale + positions = torch.randint(0, cos_sin.shape[0], (rows,), device="cuda", dtype=torch.int32) + query = low_m_gemm.mxfp8_lowm_linear( + qr, + linear.weight, + linear.weight_scale, + rope=(positions, cos_sin, 128), + scale_swizzled=True, + ) + expected = quantization.pack_index_queries_split(query.view(rows, 32, 128)) + actual = low_m_gemm.mxfp8_lowm_index_query( + qr, linear.weight, linear.weight_scale, positions, cos_sin + ) + for got, want in zip(actual, expected): + assert torch.equal(got.view(torch.uint8), want.view(torch.uint8)), rows + + manager, request = module_cache + model, _ = _native_model( + aux_stream=torch.cuda.Stream(), layout=manager.layout, kv_cache_dtype="fp8" + ) + for linear in (model.wq_a, model.wq_b, model.wkv): + linear.quant_method.tune_mxfp8_backends = True + monkeypatch.setattr(csa2_module, "do_multi_stream", lambda: True) + side_streams = [] + side_stream = csa2_module.csa2_side_stream + monkeypatch.setattr( + csa2_module, + "csa2_side_stream", + lambda device, index=0: side_streams.append(index) or side_stream(device, index), + ) + packed_queries = [] + store = cache_manager.store_layer_rows + monkeypatch.setattr( + cache_manager, + "store_layer_rows", + lambda *args, index_q=None, **kwargs: packed_queries.append(index_q is not None) + or store(*args, index_q=index_q, **kwargs), + ) + metadata = CSA2TrtllmMetadata(max_num_requests=1, max_num_tokens=8, kv_cache_manager=manager) + metadata.request_ids = [request.py_request_id] + metadata.num_contexts = 1 + metadata.prompt_lens = [4] + metadata.seq_lens = torch.tensor([1], dtype=torch.int32, device="cpu") + hidden = torch.randn(1, 128, device="cuda", dtype=torch.bfloat16) + metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[0]) + metadata.prepare() + model(hidden, metadata.csa2_positions, metadata) + metadata.kv_cache_params = KVCacheParams(use_cache=True, num_cached_tokens_per_seq=[1]) + metadata.num_contexts = 0 + metadata.is_cuda_graph = True + metadata.prepare() + positions = metadata.csa2_positions + pools = (manager.get_swa_buffer(0), manager.get_main_buffer(0), manager.get_index_pages(0)) + torch.cuda.synchronize() + base = [pool.clone() for pool in pools] + + packs_index_q = csa2_module.DeepseekV41Attention._packs_index_q + + def run(pack): + monkeypatch.setattr( + csa2_module.DeepseekV41Attention, + "_packs_index_q", + packs_index_q if pack else (lambda self, rows, positions: False), + ) + for pool, saved in zip(pools, base): + pool.copy_(saved) + metadata.reset_routing() + return model(hidden, positions, metadata) + + def observe(output): + torch.cuda.synchronize() + return [output.clone(), metadata.csa2_indices[0].clone(), *(p.clone() for p in pools)] + + with flashinfer_mxfp8_decode_graph_capture(): + with flashinfer_mxfp8_autotune(): + run(True) + for _ in range(2): + hidden.normal_() + side_streams.clear() + packed_queries.clear() + expected = observe(run(False)) + assert packed_queries == [True] and 4 not in side_streams + side_streams.clear() + packed_queries.clear() + actual = observe(run(True)) + assert packed_queries == [False] and 4 in side_streams + for got, want in zip(actual, expected): + assert torch.equal(got, want) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run(True) + for sign in (-1.0, 1.0): + hidden.mul_(sign) + for pool, saved in zip(pools, base): + pool.copy_(saved) + graph.replay() + actual = observe(captured) + expected = observe(run(False)) + for got, want in zip(actual, expected): + assert torch.equal(got, want) + torch.cuda.synchronize() + + @skip_pre_blackwell @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("module_cache", [2], indirect=True) diff --git a/tests/unittest/_torch/multi_gpu/test_allreduce.py b/tests/unittest/_torch/multi_gpu/test_allreduce.py index ae85f896acf0..a8e51ada6c47 100644 --- a/tests/unittest/_torch/multi_gpu/test_allreduce.py +++ b/tests/unittest/_torch/multi_gpu/test_allreduce.py @@ -36,9 +36,11 @@ from tensorrt_llm._torch.distributed.mnnvl_memory import MnnvlMemory from tensorrt_llm._torch.distributed.ops import get_allreduce_workspace from tensorrt_llm._torch.modules.linear import Linear, TensorParallelMode +from tensorrt_llm._torch.modules.low_m_gemm import mxfp8_lowm_linear from tensorrt_llm._torch.modules.mhc.mhc_lowm import ( LamportAllReduce, MnnvlLamportAllReduce, MoeUnfinalized, - mhc_allreduce_post_collapse_norm, mhc_post_collapse_norm) + defer_ob_projection, mhc_allreduce_post_collapse_norm, + mhc_post_collapse_norm) from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.nccl_window_tensor_scope import \ discard_nccl_window_tensor_outputs @@ -1187,6 +1189,47 @@ def _mhc_moe_inputs(m: int, seed: int, rank: int, top_k: int = 6): return parts, res, post, comb, pre, weight +def _mhc_ob_inputs(m: int, seed: int, rank: int, k: int = 2048): + """This rank's o_b input and MXFP8 weight (UE8M0 scales, linear layout) and the boundary operands.""" + _, res, post, comb, pre, weight = _mhc_inputs(m, seed, rank) + gen = torch.Generator(device="cuda").manual_seed(seed * 100 + rank + 11) + o_lat = torch.randn(m, k, generator=gen, device="cuda").bfloat16() + ob_weight = (torch.randn(MHC_HIDDEN, k, generator=gen, device="cuda") * + 0.5).to(torch.float8_e4m3fn) + ob_scale = torch.randint(118, + 126, (MHC_HIDDEN, k // 32), + generator=gen, + device="cuda", + dtype=torch.uint8) + return o_lat, ob_weight, ob_scale, res, post, comb, pre, weight + + +def _flat(outputs): + for out in outputs: + if isinstance(out, tuple): + yield from out + else: + yield out + + +def _mhc_allreduce_and_lamport(tp_size: int, mnnvl: bool, rank: int): + """The library all-reduce and the fused boundary's Lamport workspace of one protocol.""" + mapping = Mapping(world_size=tp_size, tp_size=tp_size, rank=rank) + if mnnvl: + # The MNNVL protocol on one node (fabric memory replaced by POSIX handles). + os.environ["TLLM_TEST_MNNVL"] = "1" + allreduce = AllReduce(mapping=mapping, + strategy=AllReduceStrategy.MNNVL, + dtype=torch.bfloat16) + assert allreduce.mnnvl_allreduce is not None + return allreduce, MnnvlLamportAllReduce(mapping) + allreduce = AllReduce(mapping=mapping, + strategy=AllReduceStrategy.ONESHOT, + dtype=torch.bfloat16) + return allreduce, LamportAllReduce(get_allreduce_workspace(mapping), rank, + tp_size) + + # Pool workers run later tests: restore the environment. @patch.dict(os.environ) @torch.inference_mode() @@ -1196,21 +1239,7 @@ def run_mhc_allreduce_boundary_single_rank(tp_size: int, mnnvl: bool) -> bool: rank = tensorrt_llm.mpi_rank() torch.cuda.set_device(rank) try: - mapping = Mapping(world_size=tp_size, tp_size=tp_size, rank=rank) - if mnnvl: - # The MNNVL protocol on one node (fabric memory replaced by POSIX handles). - os.environ["TLLM_TEST_MNNVL"] = "1" - allreduce = AllReduce(mapping=mapping, - strategy=AllReduceStrategy.MNNVL, - dtype=torch.bfloat16) - assert allreduce.mnnvl_allreduce is not None - lamport = MnnvlLamportAllReduce(mapping) - else: - allreduce = AllReduce(mapping=mapping, - strategy=AllReduceStrategy.ONESHOT, - dtype=torch.bfloat16) - lamport = LamportAllReduce(get_allreduce_workspace(mapping), rank, - tp_size) + allreduce, lamport = _mhc_allreduce_and_lamport(tp_size, mnnvl, rank) def reference(x, res, post, comb, pre, weight): return mhc_post_collapse_norm(allreduce(x), res, post, comb, pre, @@ -1250,11 +1279,12 @@ def fused(x, res, post, comb, pre, weight): for g, w in zip(got, want): torch.testing.assert_close(g, w, rtol=0, atol=0) - # The MoE-finalize input. - parts, *args = _mhc_moe_inputs(24, 324, rank) - want = reference(parts.finalize(), *args) - for g, w in zip(fused(parts, *args), want): - torch.testing.assert_close(g, w, rtol=0, atol=0) + # The MoE-finalize input, with a cluster of CTAs per token (6 rows) and one CTA per token. + for m in (6, 24): + parts, *args = _mhc_moe_inputs(m, 300 + m, rank) + want = reference(parts.finalize(), *args) + for g, w in zip(fused(parts, *args), want): + torch.testing.assert_close(g, w, rtol=0, atol=0) # Contiguous MoE views can have storage offsets that violate CuTe's alignment. def offset_copy(value): @@ -1290,6 +1320,63 @@ def offset_copy(value): return True +# Pool workers run later tests: restore the environment. +@patch.dict(os.environ) +@torch.inference_mode() +def run_mhc_ob_boundary_single_rank(tp_size: int, mnnvl: bool) -> bool: + """The attention's o_b projected in the boundary's launch vs the o_b GEMM then the boundary: + the same bits, also with the NVFP4 copy of the output and in a replayed graph.""" + rank = tensorrt_llm.mpi_rank() + torch.cuda.set_device(rank) + try: + allreduce, lamport = _mhc_allreduce_and_lamport(tp_size, mnnvl, rank) + nvfp4_scale = torch.tensor([0.37], device="cuda") + for m in (1, 6, 16): + o_lat, ob_weight, ob_scale, *args = _mhc_ob_inputs(m, 500 + m, rank) + deferred = defer_ob_projection(o_lat, ob_weight, ob_scale) + assert deferred is not None + partial = mxfp8_lowm_linear(o_lat, ob_weight, ob_scale) + for scale in (None, nvfp4_scale): + want = mhc_allreduce_post_collapse_norm(partial, + *args, + MHC_NORM_EPS, + lamport, + nvfp4_scale=scale) + got = mhc_allreduce_post_collapse_norm(deferred, + *args, + MHC_NORM_EPS, + lamport, + nvfp4_scale=scale) + for g, w in zip(_flat(got), _flat(want)): + assert torch.equal(g.view(torch.uint8), w.view(torch.uint8)) + # Replays of the o_b boundary between the MoE-finalize boundary and a library all-reduce + # on one workspace: whichever CTA arrives last hands the triple buffer over. + o_lat, ob_weight, ob_scale, *args = _mhc_ob_inputs(6, 507, rank) + parts, *moe_args = _mhc_moe_inputs(6, 508, rank) + lib_in = _mhc_inputs(6, 509, rank)[0] + + def launches(deferred): + return (*mhc_allreduce_post_collapse_norm(parts, *moe_args, + MHC_NORM_EPS, lamport), + *mhc_allreduce_post_collapse_norm(deferred, *args, + MHC_NORM_EPS, lamport), + allreduce(lib_in)) + + want = launches(mxfp8_lowm_linear(o_lat, ob_weight, ob_scale)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = launches(defer_ob_projection(o_lat, ob_weight, ob_scale)) + for _ in range(50): + graph.replay() + torch.cuda.synchronize() + for g, w in zip(got, want): + torch.testing.assert_close(g, w, rtol=0, atol=0) + except Exception: + traceback.print_exc() + raise + return True + + @pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.device_count() < 2 or torch.cuda.get_device_capability() not in ((10, 0), (10, 3), (10, 7)), @@ -1307,3 +1394,120 @@ def test_mhc_allreduce_boundary_matches_allreduce_then_boundary( *zip(*[(tp_size, mnnvl)] * tp_size)) for r in results: assert r is True + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.device_count() < 2 + or torch.cuda.get_device_capability() not in ((10, 0), (10, 3)), + reason="needs 2 SM100/SM103 GPUs", +) +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_mhc_ob_boundary_matches_ob_then_boundary(mpi_pool_executor): + if not MnnvlMemory.supports_mnnvl(): + pytest.skip("needs MNNVL memory") + tp_size = mpi_pool_executor.num_workers + results = mpi_pool_executor.map(run_mhc_ob_boundary_single_rank, + *zip(*[(tp_size, True)] * tp_size)) + for r in results: + assert r is True + + +# Pool workers run later tests: restore the environment. +@patch.dict(os.environ) +@torch.inference_mode() +def run_mhc_moe_boundary_behind_lowm_fc2_single_rank(tp_size: int): + """Captured layers of [attention boundary -> FC1-only MoE -> CuTe FC2 -> MoE boundary] on one + MNNVL workspace, the MoE boundaries prefetching an o_b weight into L2, give the eager bits.""" + from _torch.moe.test_moe_lowm_fc2 import (_DSV41_EP_RANK, + _DSV41_ROUTED_SCALE, + _bitwise_tactics, _dsv41_inputs) + + from tensorrt_llm._torch.custom_ops.trtllm_gen_custom_ops import \ + FP4BlockScaleMoERunner + from tensorrt_llm._torch.models.modeling_deepseekv41 import \ + _MOE_CUTE_FC2_ROWS + + rank = tensorrt_llm.mpi_rank() + torch.cuda.set_device(rank) + try: + _, lamport = _mhc_allreduce_and_lamport(tp_size, True, rank) + m, shape = 6, _DSV41_EP_RANK + top_k, local_offset = shape.top_k, rank * shape.local + inputs, _ = _dsv41_inputs(m, shape, local_offset, 900 + rank) + # Given routing weights, the runner uses them and leaves its own unwritten. + topk_weights = inputs[16] + runner = FP4BlockScaleMoERunner(shape.num_experts, + top_k, + None, + None, + shape.inter, + local_offset, + shape.local, + _DSV41_ROUTED_SCALE, + 1, + False, + 0, + cute_fc2_rows=_MOE_CUTE_FC2_ROWS) + tactic = _bitwise_tactics(runner.get_runner(), m, shape)[0] + layers = [(_mhc_inputs(m, 40 + i, rank), _mhc_inputs(m, 60 + i, rank)) + for i in range(3)] + ob_weights = (torch.zeros(5120, + 1024, + dtype=torch.float8_e4m3fn, + device="cuda"), + torch.zeros(5120, 32, dtype=torch.uint8, device="cuda")) + + def chain(prefetch): + outs = [] + for attn, ffn in layers: + outs.append( + mhc_allreduce_post_collapse_norm(*attn, MHC_NORM_EPS, + lamport)) + fc2, _, perm = runner.forward(inputs, tactic=list(tactic)) + parts = MoeUnfinalized(shared=ffn[0], + fc2=fc2, + scales=topk_weights, + perm=perm.flatten()[:m * top_k].view( + m, top_k)) + outs.append( + mhc_allreduce_post_collapse_norm(parts, + *ffn[1:], + MHC_NORM_EPS, + lamport, + l2_prefetch=prefetch)) + return list(_flat(outs)) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + want = chain(()) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = chain(ob_weights) + for _ in range(20): + graph.replay() + torch.cuda.synchronize() + for g, w in zip(got, want): + assert torch.equal(g.view(torch.uint8), w.view(torch.uint8)) + except Exception: + traceback.print_exc() + raise + return True + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.device_count() < 2 + or torch.cuda.get_device_capability() not in ((10, 0), (10, 3)), + reason="needs 2 SM100/SM103 GPUs", +) +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_mhc_moe_boundary_behind_lowm_fc2(mpi_pool_executor): + if not MnnvlMemory.supports_mnnvl(): + pytest.skip("needs MNNVL memory") + tp_size = mpi_pool_executor.num_workers + results = list( + mpi_pool_executor.map(run_mhc_moe_boundary_behind_lowm_fc2_single_rank, + *zip(*[(tp_size, )] * tp_size))) + for r in results: + assert r is True