Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/tensorrt_llm/kernels/fusedCatFp4.cu
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ __global__ __launch_bounds__(WARP_SIZE* ROWS_PER_BLOCK) void fusedCatFp4Kernel(i
uint8_t byte1 = static_cast<uint8_t>(c2 | (c3 << 4));

// ---- Stage 6: Store packed bytes (2 bytes per lane) ----
int base_out = row * (HEAD_DIM / 2) + lane * 2;
int64_t const base_out = static_cast<int64_t>(row) * (HEAD_DIM / 2) + lane * 2;
packed_out[base_out + 0] = static_cast<int8_t>(byte0);
packed_out[base_out + 1] = static_cast<int8_t>(byte1);

Expand Down
2 changes: 1 addition & 1 deletion cpp/tensorrt_llm/kernels/fusedCatFp8.cu
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ __global__ __launch_bounds__(WARP_SIZE* ROWS_PER_BLOCK) void fusedCatFp8Kernel(_
packed.fp8[2] = quantize(v2);
packed.fp8[3] = quantize(v3);

int base_out = row * HEAD_DIM + lane * ELEMS_PER_THREAD;
int64_t const base_out = static_cast<int64_t>(row) * HEAD_DIM + lane * ELEMS_PER_THREAD;
*reinterpret_cast<uint32_t*>(fp8_out + base_out) = packed.u32;

if (lane == 0)
Expand Down
16 changes: 13 additions & 3 deletions tensorrt_llm/_torch/attention/backends/sparse/csa2/kernel.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,12 @@ def floor_mod(a: Int64, b: Int64) -> Int64:

@cute.jit
def tensor1d(ptr: cute.Pointer, count: Int64):
return cute.make_tensor(ptr, cute.make_layout(count))
# Keep offsets beyond 2 GiB in 64 bits on CuTe DSL builds with 32-bit default strides.
return cute.make_tensor(ptr, cute.make_layout(count, stride=Int64(1)))

@cute.jit
def tensor2d(ptr: cute.Pointer, rows: Int64, cols: Int64, row_stride: Int64):
return cute.make_tensor(ptr, cute.make_layout((rows, cols), stride=(row_stride, 1)))
return cute.make_tensor(ptr, cute.make_layout((rows, cols), stride=(row_stride, Int64(1))))

@cute.jit
def _int32_ptr(address: Int64):
Expand Down Expand Up @@ -7825,13 +7826,22 @@ def _mapping_args(main_mapping, main_slots, main_logical, count: int, device):
``(page_table, page_size, max_positions, requests, visible)``; otherwise
``main_slots`` already hold physical rows.
"""
from tensorrt_llm._torch.memory_buffer_utils import get_memory_buffers

identity = _constant_zeros(device, (1, 1), torch.int32)
if main_mapping is None:
_check(main_slots is not None, "CSA2 staging requires physical main slots or a mapping")
slots = main_slots if main_slots.dtype == torch.int64 else main_slots.long()
logical = main_logical if main_logical is not None else slots
table_args = (_ptr(identity), 1, 1, 1, 1, 2**62)
zeros = _constant_zeros(device, (count,), torch.int64)
# Pooled rather than a constant per shape: the row count changes from step to step.
zeros = get_memory_buffers().get_buffer(
[count],
torch.int64,
buffer_name="csa2_unmapped_rows",
reserve_buffer=torch.cuda.is_current_stream_capturing(),
Comment thread
ziyixiong-nv marked this conversation as resolved.
Resolved
)
zeros.zero_()
return (
True,
logical,
Expand Down
6 changes: 4 additions & 2 deletions tensorrt_llm/_torch/kernels/blackwell/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@
from cutlass.cutlass_dsl import T, dsl_user_op

TRTLLM_ENABLE_PDL = os.environ.get("TRTLLM_ENABLE_PDL", "1") == "1"
# A memory clobber, as in CUTLASS C++: no memory access moves across the asm.
_PDL_ASM_CONSTRAINTS = "~{memory}"


# WAR for CuTeDSL make_ptr implementation
Expand Down Expand Up @@ -471,7 +473,7 @@ def griddepcontrol_wait(*, loc=None, ip=None) -> None:
res=None,
operands_=[],
asm_string="griddepcontrol.wait;",
constraints="",
constraints=_PDL_ASM_CONSTRAINTS,
has_side_effects=True,
asm_dialect=llvm.AsmDialect.AD_ATT,
loc=loc,
Expand All @@ -491,7 +493,7 @@ def griddepcontrol_launch_dependents(*, loc=None, ip=None) -> None:
res=None,
operands_=[],
asm_string="griddepcontrol.launch_dependents;",
constraints="",
constraints=_PDL_ASM_CONSTRAINTS,
has_side_effects=True,
asm_dialect=llvm.AsmDialect.AD_ATT,
loc=loc,
Expand Down
7 changes: 7 additions & 0 deletions tensorrt_llm/_torch/modules/mhc/mhc_lowm.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,13 @@ def mhc_allreduce_post_collapse_norm(
res_out = torch.empty_like(res_flat)
quant, quantized, scale_addr = _quantized_outputs(m, hidden, out.device, mxfp8, nvfp4_scale)
table = allreduce.prepare(allreduce.nranks * m * hidden * res_flat.element_size())
# A contiguous offset view can still violate the compiled kernel's 16-byte alignment.
res_flat, post, comb, pre, weight, fc2, scales, perm, shared = [
t.clone() if t.data_ptr() % 16 else t
for t in (res_flat, post, comb, pre, weight, fc2, scales, perm, shared)
]
# Unfinalized MoE uses shared as x too; keep the alias to avoid a second copy.
x = shared.view(m, hidden) if moe_key is not None else (x.clone() if x.data_ptr() % 16 else x)
fn = _compiled_allreduce_boundary(
m,
hidden,
Expand Down
29 changes: 29 additions & 0 deletions tests/unittest/_torch/multi_gpu/test_allreduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -1255,6 +1255,35 @@ def fused(x, res, post, comb, pre, weight):
want = reference(parts.finalize(), *args)
for g, w in zip(fused(parts, *args), want):
torch.testing.assert_close(g, w, rtol=0, atol=0)

# Contiguous MoE views can have storage offsets that violate CuTe's alignment.
def offset_copy(value):
storage = torch.empty(value.numel() + 1,
dtype=value.dtype,
device=value.device)
view = storage[1:].view(value.shape)
view.copy_(value)
assert view.is_contiguous() and view.data_ptr() % 16
return view

parts = MoeUnfinalized(
**{
name: offset_copy(getattr(parts, name))
for name in ('shared', 'fc2', 'scales', 'perm')
})
want = reference(parts.finalize(), *args)
for g, w in zip(fused(parts, *args), want):
torch.testing.assert_close(g, w, rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = fused(parts, *args)
parts.shared.mul_(.875)
want = reference(parts.finalize(), *args)
for _ in range(32):
graph.replay()
torch.cuda.synchronize()
for g, w in zip(captured, want):
torch.testing.assert_close(g, w, rtol=0, atol=0)
except Exception:
traceback.print_exc()
raise
Expand Down
Loading