Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tensorrt_llm/_torch/pyexecutor/llm_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -1487,6 +1487,7 @@ def create_child_request(self, child_id: int) -> None:
py_multimodal_data=deepcopy(self.py_multimodal_data),
encoder_input_features=self.py_encoder_input_features,
locality_domain_id=self.py_locality_domain_id,
_validation_error=self._validation_error,
)
child.py_bad_words = deepcopy(self.py_bad_words)
child.py_no_repeat_ngram_size = self.py_no_repeat_ngram_size
Expand Down
2 changes: 2 additions & 0 deletions tensorrt_llm/_torch/pyexecutor/py_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -8809,6 +8809,8 @@ def _validate_embedding_bias(self, request: LlmRequest) -> None:
raise ValueError(
f"Embedding bias index must be in [0, {self._num_embeddings - 1}]"
)
if not math.isfinite(val):
raise ValueError("Embedding bias values must be finite")

def _validate_encoder_output_len(self, request: LlmRequest) -> None:
"""Reject an encoder input that no iteration can serve.
Expand Down
35 changes: 35 additions & 0 deletions tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py
Original file line number Diff line number Diff line change
Expand Up @@ -617,3 +617,38 @@ def apply_batched_occurrence_penalties(
max_beam_width,
fold_pending,
)

@staticmethod
@torch.compile(
dynamic=None,
fullgraph=True,
options=dict(
split_reductions=False,
),
)
def _gather_masked_inplace_add_impl(
summands_cuda: torch.Tensor,
summand_indices_cuda: torch.Tensor,
output_cuda: torch.Tensor,
output_mask_cuda: torch.Tensor,
) -> None:
output_cuda[output_mask_cuda, ...] += torch.index_select(
summands_cuda, 0, summand_indices_cuda
)

@staticmethod
def gather_masked_inplace_add(
summands_cuda: torch.Tensor,
summand_indices_cuda: torch.Tensor,
output_cuda: torch.Tensor,
output_mask_cuda: torch.Tensor,
) -> None:
# NB: helper function for apply_embedding_bias, torch.compile is expected to reduce
# data movement.
torch._dynamo.mark_dynamic(summands_cuda, 0)
torch._dynamo.mark_dynamic(summand_indices_cuda, 0)
torch._dynamo.mark_dynamic(output_cuda, 0)
torch._dynamo.mark_dynamic(output_mask_cuda, 0)
Fusions._gather_masked_inplace_add_impl(
summands_cuda, summand_indices_cuda, output_cuda, output_mask_cuda
)
18 changes: 15 additions & 3 deletions tensorrt_llm/_torch/pyexecutor/sampler/sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@
)
from .sampler_features import (
AsyncWorkerMixin,
EmbeddingBiasCache,
SamplerEvent,
_PackedStepIndexer,
_UnpackedStepIndexer,
apply_embedding_bias,
check_stop_words_length,
fast_greedy_sample_kernel,
meet_stop_token_criteria,
Expand Down Expand Up @@ -485,6 +485,8 @@ def __init__(self, args: Args):
# constructed before _create_store, which reads the shape.
self._log_probs = LogProbsHandler(self)

self._embedding_bias_cache: EmbeddingBiasCache | None = None

# The Torch sampler hard-depends on flashinfer. Enforce it once here, at
# construction, so the check stays out of the CUDA-graph-captured
# sampling loop.
Expand Down Expand Up @@ -2647,8 +2649,18 @@ def _process_requests(
seq_lens_cuda = seq_lens_host.to(device="cuda", non_blocking=True)

# Handle embedding bias
apply_embedding_bias(
logits_cuda, sampling_requests, sampling_requests_metadata.req_num_steps
if self._embedding_bias_cache is None:
# Always initializing to ensure correct KV cache sizing
self._embedding_bias_cache = EmbeddingBiasCache(
logit_dtype=logits_cuda.dtype,
vocab_size=logits_cuda.size(-1),
max_seq_slots=self.max_num_sequences,
device=logits_cuda.device,
)
self._embedding_bias_cache.apply_bias(
logits_cuda,
sampling_requests,
sampling_requests_metadata.req_num_steps,
)

# Apply repetition/presence/frequency penalties in place, before the greedy fast
Expand Down
277 changes: 172 additions & 105 deletions tensorrt_llm/_torch/pyexecutor/sampler/sampler_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
from tensorrt_llm.sampling_params import EmbeddingBias

from ..llm_request import LlmRequest
from .ops.vanilla import Fusions
from .sampler_common import DEFAULT_BEAM_IDX

if sys.version_info[:2] >= (3, 12):
Expand All @@ -68,7 +69,7 @@
"_StepIndexTranslator",
"_StridedStepIndexTranslator",
"_UnpackedStepIndexer",
"apply_embedding_bias",
"EmbeddingBiasCache",
"check_stop_words_length",
"fast_greedy_sample_kernel",
"first_stop_reason",
Expand Down Expand Up @@ -512,114 +513,180 @@ def fast_greedy_sample_kernel(
return next_tokens


def apply_embedding_bias(
logits: torch.Tensor,
requests: list[LlmRequest],
request_steps: torch.Tensor,
) -> None:
"""Apply embedding bias (aka logit bias) to logits.
class EmbeddingBiasCache:
def __init__(
self, *, logit_dtype: torch.dtype, vocab_size: int, max_seq_slots: int, device: torch.device
):
self._flat_cache = torch.empty(
(max_seq_slots, vocab_size), dtype=logit_dtype, device=device
)
Comment thread
ixlmar marked this conversation as resolved.
Resolved
self._device = device
self._logit_dtype = logit_dtype
self._vocab_size = vocab_size

# Slots can be reused, but request_id is quasi unique (cf. MIN_GLOBAL_ID).
# Tracking which request_id is cached in each slot.
self._slot_to_request_id: dict[int, int] = {}

def _is_cached(self, req: LlmRequest) -> bool:
# NB: LlmRequest.py_request_id is set from .request_id, which is an integer.
return self._slot_to_request_id.get(req.py_seq_slot) == req.py_request_id

def _mark_cached(self, req: LlmRequest) -> None:
self._slot_to_request_id[req.py_seq_slot] = req.py_request_id

def _update(self, reqs: list[LlmRequest]) -> None:
# NB: This could be extended to allow reuse of equivalent rows in _flat_cache
# (potentially relevant for output sequence length <~ batch size).

reqs_needing_update = [
req for req in reqs if req.py_embedding_bias and not self._is_cached(req)
]

if not reqs_needing_update:
return

_next_bias_index = 0

def provision_bias_index() -> int:
nonlocal _next_bias_index
bias_index = _next_bias_index
_next_bias_index += 1
return bias_index

# Indices of unique bias rows
bias_to_index: dict[EmbeddingBias, int] = defaultdict(provision_bias_index)

# Source indices for bias cache update
bias_gather_indices: list[int] = []
# Destination indices for bias cache update
bias_scatter_indices: list[int] = []

# Collect bias information
for req in reqs_needing_update:
req_bias = req.py_embedding_bias
assert req_bias is not None
bias_gather_indices.append(bias_to_index[req_bias])
seq_slot = req.py_seq_slot
assert seq_slot is not None
bias_scatter_indices.append(seq_slot)

bias_gather_indices_cuda = torch.tensor(
bias_gather_indices, pin_memory=prefer_pinned(), dtype=torch.int32
).to(self._device, non_blocking=True)
bias_scatter_indices_cuda = torch.tensor(
bias_scatter_indices, pin_memory=prefer_pinned(), dtype=torch.int32
).to(self._device, non_blocking=True)

unique_biases_tensor_cuda = torch.zeros(
(len(bias_to_index), self._vocab_size),
dtype=self._logit_dtype,
device=self._device,
)
unique_biases_flat_idx_cuda = torch.tensor(
[
self._vocab_size * unique_bias_idx + logit_idx
for sparse_bias, unique_bias_idx in bias_to_index.items()
for (logit_idx, bias_value) in sparse_bias
],
pin_memory=prefer_pinned(),
dtype=torch.int32,
).to(self._device, non_blocking=True)
unique_biases_flat_values_cuda = torch.tensor(
[
bias_value
for sparse_bias, unique_bias_idx in bias_to_index.items()
for (logit_idx, bias_value) in sparse_bias
],
pin_memory=prefer_pinned(),
dtype=self._logit_dtype,
).to(self._device, non_blocking=True)

unique_biases_tensor_cuda.view(-1).scatter_(
dim=0,
index=unique_biases_flat_idx_cuda,
src=unique_biases_flat_values_cuda,
)

Arguments:
request_steps: Number of steps/tokens for each request.
self._flat_cache[bias_scatter_indices_cuda, :] = unique_biases_tensor_cuda[
bias_gather_indices_cuda, :
]

Modifies logits in-place.
"""
# NB: Unfortunately, Torch provides no combination of torch.index_select (similar to
# torch.Tensor.gather -- allows one-to-many mapping) and addition, analogous to how
# torch.Tensor.scatter_add_ (and its variant torch.Tensor.index_add_ -- allows
# many-to-one mapping) combine addition with torch.Tensor.scatter_.
#
# Notwithstanding the previous point, there are two options:
# (i) materialize a permuted bias tensor with repeated consecutive rows via
# torch.repeat_interleave and then use torch.Tensor.index_add_ (poor write
# locality / risk of false sharing)
# (ii) materialize the correctly ordered bias tensor via torch.index_select and then
# perform a masked addition (poor read locality for request batches randomly
# mixing uniform and heterogeneous bias tensors, i.e., mixing slices with high
# and low reuse).
# Since read-caching is expected to help in typical cases, option (ii) is implemented here.

# Track which logits require logit bias application
request_steps_list = request_steps.tolist()
logits_bias_masks = [False] * logits.size(0)
_next_bias_index = 0

def provision_bias_index() -> int:
nonlocal _next_bias_index
bias_index = _next_bias_index
_next_bias_index += 1
return bias_index

# Indices of unique bias rows
bias_to_index: dict[EmbeddingBias, int] = defaultdict(provision_bias_index)

# Source indices for bias application
bias_gather_indices: list[int] = []

# Collect bias information
#
# NB: the mask indexes rows of 'logits', not requests: a request contributes
# 'steps' consecutive rows (> 1 under speculative decoding), so the offset
# has to accumulate 'steps' rather than track the request index. It must
# stay in step with 'bias_gather_indices', which is built in row order.
row_offset = 0
for req, steps in zip(requests, request_steps_list):
req_bias = req.py_embedding_bias
if req_bias is not None:
for j in range(row_offset, row_offset + steps):
logits_bias_masks[j] = True
req_bias_index = bias_to_index[req_bias]
bias_gather_indices.extend(repeat(req_bias_index, steps))
row_offset += steps

if not bias_to_index:
return

bias_gather_indices_cuda = torch.tensor(
bias_gather_indices, pin_memory=prefer_pinned(), dtype=torch.int32
).to(logits.device, non_blocking=True)
logits_bias_mask_cuda = torch.tensor(
logits_bias_masks, pin_memory=prefer_pinned(), dtype=torch.bool
).to(logits.device, non_blocking=True)

vocab_size = logits.size(-1)
unique_biases_tensor_cuda = torch.zeros(
(len(bias_to_index), vocab_size),
dtype=logits.dtype,
device=logits.device,
)
unique_biases_flat_idx_cuda = torch.tensor(
[
vocab_size * unique_bias_idx + logit_idx
for sparse_bias, unique_bias_idx in bias_to_index.items()
for (logit_idx, bias_value) in sparse_bias
],
pin_memory=prefer_pinned(),
dtype=torch.int32,
).to(logits.device, non_blocking=True)
unique_biases_flat_values_cuda = torch.tensor(
[
bias_value
for sparse_bias, unique_bias_idx in bias_to_index.items()
for (logit_idx, bias_value) in sparse_bias
],
pin_memory=prefer_pinned(),
dtype=logits.dtype,
).to(logits.device, non_blocking=True)
unique_biases_tensor_cuda.view(-1).scatter_(
dim=0,
index=unique_biases_flat_idx_cuda,
src=unique_biases_flat_values_cuda,
)
for req in reqs_needing_update:
self._mark_cached(req)

biases_tensor_cuda = torch.index_select(unique_biases_tensor_cuda, 0, bias_gather_indices_cuda)
def apply_bias(
self,
logits: torch.Tensor,
requests: list[LlmRequest],
request_steps: torch.Tensor,
) -> None:
"""Apply embedding bias (aka logit bias) to logits.

# NB: Avoiding logits[bias_scatter_indices] += biases_tensor (and torch.Tensor.scatter_add_), because it
# is unclear if this allows for repeated indices, cf.
# https://docs.pytorch.org/docs/2.8/generated/torch.Tensor.index_put_.html#torch-tensor-index-put
# and thus introduces read-after-write dependencies (including possible false
# sharing).
logits[logits_bias_mask_cuda] += biases_tensor_cuda
Arguments:
request_steps: Number of steps/tokens for each request.

Modifies logits in-place.
"""
# NB: Unfortunately, Torch provides no combination of torch.index_select (similar to
# torch.Tensor.gather -- allows one-to-many mapping) and addition, analogous to how
# torch.Tensor.scatter_add_ (and its variant torch.Tensor.index_add_ -- allows
# many-to-one mapping) combine addition with torch.Tensor.scatter_.
#
# Notwithstanding the previous point, there are two options:
# (i) materialize a permuted bias tensor with repeated consecutive rows via
# torch.repeat_interleave and then use torch.Tensor.index_add_ (poor write
# locality / risk of false sharing)
# (ii) materialize the correctly ordered bias tensor via torch.index_select and then
# perform a masked addition (poor read locality for request batches randomly
# mixing uniform and heterogeneous bias tensors, i.e., mixing slices with high
# and low reuse).
# Since read-caching is expected to help in typical cases, option (ii) is implemented here.

if all(not req.py_embedding_bias for req in requests):
return

# Track which logits require logit bias application
request_steps_list = request_steps.tolist()
logits_bias_masks = [False] * logits.size(0)

# Source indices for bias application
bias_gather_indices: list[int] = []

# Collect bias information
#
# NB: the mask indexes rows of 'logits', not requests: a request contributes
# 'steps' consecutive rows (> 1 under speculative decoding), so the offset
# has to accumulate 'steps' rather than track the request index.
row_offset = 0
reqs_using_bias = []
for req, steps in zip(requests, request_steps_list):
req_bias = req.py_embedding_bias
if req_bias:
reqs_using_bias.append(req)
for j in range(row_offset, row_offset + steps):
logits_bias_masks[j] = True
seq_slot = req.py_seq_slot
assert seq_slot is not None
bias_gather_indices.extend(repeat(seq_slot, steps))
row_offset += steps

self._update(reqs_using_bias)
biases_by_slot_cuda = self._flat_cache # indexed by seq_slot

bias_gather_indices_cuda = torch.tensor(
bias_gather_indices, pin_memory=prefer_pinned(), dtype=torch.int32
).to(logits.device, non_blocking=True)
logits_bias_mask_cuda = torch.tensor(
logits_bias_masks, pin_memory=prefer_pinned(), dtype=torch.bool
).to(logits.device, non_blocking=True)

Fusions.gather_masked_inplace_add(
summands_cuda=biases_by_slot_cuda,
summand_indices_cuda=bias_gather_indices_cuda,
output_cuda=logits,
output_mask_cuda=logits_bias_mask_cuda,
)


# --------------------------------------------------------------------------
Expand Down
Loading
Loading