From 76372a755de5f8f6a43c17e2f69efbf054dc919f Mon Sep 17 00:00:00 2001 From: Bastian Hagedorn Date: Tue, 10 Nov 2026 09:11:39 +0100 Subject: [PATCH] knn: wait for the TMEM load before releasing the accumulator stage In blackwell_impl.py the four warps copy the distance tile from tensor memory to registers (cute.copy with tiled_copy_t2r, i.e. tcgen05.ld) and release the single accumulator stage (acc_pipeline.consumer_release) in the very next instruction, before any loaded register is read. Warp 0 then acquires the stage and issues the next database tile's MMA into the same columns, and only after that do the warps read the fragment for the top-K. tcgen05.ld is asynchronous: the PTX ISA (tcgen05.ld / tcgen05.wait) orders it only through tcgen05.wait::ld or a register dependency, and the mbarrier arrive has no dependency on the loaded values, so the refill MMA may overwrite the tile while the load is still reading it. The comment saying the copy had already drained the accumulator to registers was not true at the ISA level. Add cute.arch.fence_view_async_tmem_load() (tcgen05.wait::ld) between the copy and the release, in BlackwellKnnBuild and in the same loop of BlackwellKnnSearchTC, and correct the comment. On a B200 the SASS shows the arrive as the next instruction after LDTM.x64 with no scoreboard dependency on it; with warps 1-3 stalled the refill MMA was committed before their first use of the fragment in every probed tile, but the load had completed by then and no result was wrong in 900 launches. The fence adds no instruction and removes the dependency on that latency; the output is bitwise identical. Co-Authored-By: Claude Fable 5.1 --- flashlib/primitives/knn/cutedsl/blackwell_impl.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flashlib/primitives/knn/cutedsl/blackwell_impl.py b/flashlib/primitives/knn/cutedsl/blackwell_impl.py index 8b416d9..dc1f19b 100644 --- a/flashlib/primitives/knn/cutedsl/blackwell_impl.py +++ b/flashlib/primitives/knn/cutedsl/blackwell_impl.py @@ -439,6 +439,9 @@ class SharedStorage: acc_pipeline.consumer_wait(acc_cons) cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, 0, 0)], tTR_rAcc) + # tcgen05.ld is asynchronous: wait for it before the single + # accumulator stage is released to warp 0's next MMA. + cute.arch.fence_view_async_tmem_load() acc_pipeline.consumer_release(acc_cons) acc_cons.advance() @@ -446,7 +449,8 @@ class SharedStorage: # issue the next tile's MMA now (warp 0): async tcgen05 overlaps # the CUDA-core-bound top-K below. 1 acc stage suffices -- the - # copy above already drained the accumulator to registers. + # tcgen05.wait::ld above completed the copy to registers before + # the stage was released. db_next = db + 1 if dd + 1 < tiles_per_split: if warp_idx == 0: @@ -769,6 +773,9 @@ class SharedStorage: acc_pipeline.consumer_wait(acc_cons) cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, 0, 0)], tTR_rAcc) + # tcgen05.ld is asynchronous: wait for it before the single + # accumulator stage is released to warp 0's next MMA. + cute.arch.fence_view_async_tmem_load() acc_pipeline.consumer_release(acc_cons) acc_cons.advance()