Skip to content
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion flashlib/primitives/knn/cutedsl/blackwell_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,14 +439,18 @@ class SharedStorage:
acc_pipeline.consumer_wait(acc_cons)
cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, 0, 0)],
tTR_rAcc)
# tcgen05.ld is asynchronous: wait for it before the single
# accumulator stage is released to warp 0's next MMA.
cute.arch.fence_view_async_tmem_load()
acc_pipeline.consumer_release(acc_cons)
acc_cons.advance()

cute.arch.barrier() # c_sq visible to all threads

# issue the next tile's MMA now (warp 0): async tcgen05 overlaps
# the CUDA-core-bound top-K below. 1 acc stage suffices -- the
# copy above already drained the accumulator to registers.
# tcgen05.wait::ld above completed the copy to registers before
# the stage was released.
db_next = db + 1
if dd + 1 < tiles_per_split:
if warp_idx == 0:
Expand Down Expand Up @@ -769,6 +773,9 @@ class SharedStorage:
acc_pipeline.consumer_wait(acc_cons)
cute.copy(tiled_copy_t2r, tTR_tAcc[(None, None, None, 0, 0)],
tTR_rAcc)
# tcgen05.ld is asynchronous: wait for it before the single
# accumulator stage is released to warp 0's next MMA.
cute.arch.fence_view_async_tmem_load()
acc_pipeline.consumer_release(acc_cons)
acc_cons.advance()

Expand Down