Repository navigation
[https://nvbugs/7100966][fix] Profile multimodal encoders before temporary KV allocation #22468
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1220,6 +1220,8 @@ class KvCacheCreator: | |
| # it to both constructors; the managers must not re-derive it. | ||
| _joint_kv_cache_reuse = False | ||
| _gen_preemption_headroom: int = 0 | ||
| _encoder_profile_output: Optional[torch.Tensor] = None | ||
| _encoder_transient_bytes: int = 0 | ||
|
|
||
| # This rank's donation (remote_cache_config.donor), started before the | ||
| # estimation so that clients do not wait out the warmup; the final target | ||
|
|
@@ -2142,6 +2144,18 @@ def try_prepare_estimation(self) -> bool: | |
| # before budgeting, then restore it in configure_kv_cache_capacity(). | ||
| self._kv_cache_config.pool_ratio = None | ||
| estimate_max_tokens = self._get_token_num_for_estimation() | ||
| if self._dummy_encoder_inputs: | ||
| torch.cuda.empty_cache() | ||
| torch.cuda.reset_peak_memory_stats() | ||
| self._encoder_profile_output = self._encode_dummy_inputs() | ||
| torch.cuda.empty_cache() | ||
| encoder_stats = torch.cuda.memory_stats() | ||
| self._encoder_transient_bytes = ( | ||
| encoder_stats["allocated_bytes.all.peak"] - | ||
| encoder_stats["allocated_bytes.all.current"]) | ||
| logger.info( | ||
| "Multimodal encoder transient memory during profiling: " | ||
| f"{self._encoder_transient_bytes / GB:.2f} GiB") | ||
| max_tokens = min( | ||
| estimate_max_tokens, self._kv_cache_config.max_tokens | ||
| ) if self._kv_cache_config.max_tokens is not None else estimate_max_tokens | ||
|
|
@@ -2249,10 +2263,8 @@ def configure_kv_cache_capacity(self, | |
| profiled_output_bytes = 0 | ||
|
|
||
| if py_executor is not None and not self._skip_est: | ||
| # Run the MM encoder at its independent token budget, then keep the | ||
| # equivalent runtime-owned embeddings resident while the text-only | ||
| # LLM dummy fills max_num_tokens. | ||
| encoder_profile_output = self._encode_dummy_inputs() | ||
| encoder_profile_output = self._encoder_profile_output | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Required] #21052 (TRTLLM-13419, open) rewrites this same block of Whichever PR lands second has to re-derive the other's accounting; resolved as a plain text conflict, the multimodal KV budget loses either the copy-pool reservation or the early-probe transient. Please agree on the merge order with the #21052 author, port the copy-pool terms into the early probe and the new
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
||
| self._encoder_profile_output = None | ||
| if encoder_profile_output is not None: | ||
| profiled_output_bytes = (encoder_profile_output.numel() * | ||
| encoder_profile_output.element_size()) | ||
|
|
@@ -2276,8 +2288,9 @@ def configure_kv_cache_capacity(self, | |
| if response.has_error(): | ||
| raise RuntimeError(response.error_msg) | ||
|
|
||
| torch_peak_memory = torch.cuda.memory_stats( | ||
| )["allocated_bytes.all.peak"] | ||
| torch_peak_memory = max( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Nice to have] |
||
| torch.cuda.memory_stats()["allocated_bytes.all.peak"], | ||
| model_bytes + self._encoder_transient_bytes) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Side effect: the retained encoder output now sits in |
||
|
|
||
| # Release before measuring current usage so the retained | ||
| # embeddings count toward the peak but not the steady state. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,7 +8,7 @@ | |
| import os | ||
| import traceback | ||
| from concurrent.futures import ThreadPoolExecutor | ||
| from contextlib import contextmanager | ||
| from contextlib import contextmanager, nullcontext | ||
| from dataclasses import dataclass | ||
| from itertools import chain | ||
| from typing import TYPE_CHECKING, Optional, Tuple | ||
|
|
@@ -48,6 +48,7 @@ | |
| from ..attention.backends.interface import AttentionRuntimeFeatures | ||
| from ..attention.backends.utils import get_attention_backend | ||
| from ..distributed import Distributed | ||
| from ..models.modeling_multimodal_mixin import MultimodalModelMixin | ||
| from ..models.modeling_utils import timing_metric | ||
| from ..speculative import (get_num_extra_kv_tokens, get_spec_drafter, | ||
| get_spec_resource_manager, | ||
|
|
@@ -1009,7 +1010,7 @@ def allocation_scope(current_stage: ExecutorMemoryType): | |
| else: | ||
| restore_mode = sleep_config.restore_modes[current_stage] | ||
| with virtual_memory_scope(stage, restore_mode) as memory_pool: | ||
| vm_pools[stage] = memory_pool | ||
| vm_pools.setdefault(stage, []).append(memory_pool) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Nice to have] This list keeps the probe's |
||
| yield | ||
|
|
||
| with allocation_scope(ExecutorMemoryType.MODEL_ENGINE_MAIN): | ||
|
|
@@ -1428,7 +1429,11 @@ def allocation_scope(current_stage: ExecutorMemoryType): | |
| if not skip_est: | ||
| log_memory_usage("after loading weights") | ||
|
|
||
| estimating_kv_cache = kv_cache_creator.try_prepare_estimation() | ||
| estimation_scope = (allocation_scope(ExecutorMemoryType.MODEL_EXTRA) | ||
| if isinstance(model_engine.model, MultimodalModelMixin) | ||
| else nullcontext()) | ||
| with estimation_scope: | ||
| estimating_kv_cache = kv_cache_creator.try_prepare_estimation() | ||
|
|
||
| # Without the estimation, build_managers sizes the final KV cache from | ||
| # the device memory free now, so DWDP is set up first: the KV cache | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ | |
|
|
||
| import pytest | ||
| import torch | ||
| from transformers import PretrainedConfig | ||
|
|
||
| import tensorrt_llm.bindings.internal.batch_manager as batch_manager | ||
| from tensorrt_llm._errors import ResourceError | ||
|
|
@@ -52,7 +53,7 @@ | |
| KVCacheLayerSpec as KVCacheStorageLayerSpec, | ||
| ) | ||
| from tensorrt_llm._torch.pyexecutor.model_engine import ModelEngine | ||
| from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager | ||
| from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager, ResourceManagerType | ||
| from tensorrt_llm._torch.speculative.interface import SpeculativeDecodingMode | ||
| from tensorrt_llm.bindings import DataType | ||
| from tensorrt_llm.inputs.multimodal import MultimodalParams | ||
|
|
@@ -392,6 +393,141 @@ def encode_multimodal_inputs(self, multimodal_params): | |
| assert creator._dummy_encoder_inputs == [] | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| ("fraction", "decoder_workspace", "expected_quota"), | ||
| [ | ||
| pytest.param(0.5, 32 << 20, 396 << 20, id="encoder_peak"), | ||
| pytest.param(0.3, 1000 << 20, int(728 * 0.3 * (1 << 20)), id="decoder_peak_control"), | ||
| ], | ||
| ) | ||
| def test_multimodal_profiling_sizes_cache_without_a_temporary_pool( | ||
| monkeypatch, fraction, decoder_workspace, expected_quota | ||
| ): | ||
| """Price both forward peaks without requiring the encoder to fit beside the probe cache.""" | ||
| memory = SimpleNamespace(total=2048 << 20, current=128 << 20, peak=128 << 20, temporary_cache=0) | ||
|
|
||
| def record_workspace(size): | ||
| if memory.current + memory.temporary_cache + size > memory.total: | ||
| raise torch.OutOfMemoryError("Encoder workspace does not fit beside the probe cache") | ||
| memory.peak = max(memory.peak, memory.current + size) | ||
|
|
||
| class InputProcessor: | ||
| def get_mm_max_tokens_per_item(self, max_num_encoder_tokens=None): | ||
| return {"image": 4} | ||
|
|
||
| def get_dummy_mm_data(self, *, max_num_encoder_tokens, mm_counts, dtype): | ||
| return {"image": {"item_count": mm_counts["image"]}} | ||
|
|
||
| class Model(_MultimodalModel): | ||
| dtype = torch.float32 | ||
| mm_encoder = object() | ||
|
|
||
| def encode_multimodal_inputs(self, multimodal_params): | ||
| assert multimodal_params[0].multimodal_data["image"]["item_count"] == 2 | ||
| record_workspace(1000 << 20) | ||
| memory.current += 64 << 20 | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make retained-output lifetime observable. The simulated 64 MiB allocation never decreases when the output tensor is released. Both variants can therefore pass if In this test, assert that As per path instructions, flag “assertions that would pass despite a plausible regression.”
|
||
| return torch.arange(256, dtype=self.dtype).reshape(2, 128) | ||
|
|
||
| model = Model(encoder_cache_max_bytes=0) | ||
| model.model_config.pretrained_config = PretrainedConfig( | ||
| vocab_size=128, | ||
| hidden_size=128, | ||
| num_hidden_layers=32, | ||
| num_attention_heads=2, | ||
| num_key_value_heads=2, | ||
| max_position_embeddings=131072, | ||
| ) | ||
| model.model_config.attn_backend = "TRTLLM" | ||
| model_engine = SimpleNamespace( | ||
| model=model, | ||
| input_processor=InputProcessor(), | ||
| encoder_batch_size=2, | ||
| encoder_max_num_tokens=8, | ||
| batch_size=2, | ||
| max_seq_len=131072, | ||
| max_cuda_graph_batch_size=2, | ||
| attn_runtime_features=AttentionRuntimeFeatures(chunked_prefill=False), | ||
| mm_encoder_cache=None, | ||
| ) | ||
| kv_cache_config = KvCacheConfig( | ||
| free_gpu_memory_fraction=fraction, | ||
| use_kv_cache_manager_v2=True, | ||
| ) | ||
| llm_args = TorchLlmArgs(model="dummy", checkpoint_format="HF") | ||
| creator = KvCacheCreator( | ||
| model_engine=model_engine, | ||
| mapping=Mapping(), | ||
| net_max_seq_len=131072, | ||
| kv_connector_manager=None, | ||
| max_num_tokens=4, | ||
| max_beam_width=1, | ||
| tokens_per_block=64, | ||
| max_seq_len=131072, | ||
| max_batch_size=2, | ||
| kv_cache_config=kv_cache_config, | ||
| llm_args=llm_args, | ||
| speculative_config=None, | ||
| sparse_attention_config=None, | ||
| profiling_stage_data=None, | ||
| is_disagg=False, | ||
| ) | ||
|
|
||
| def memory_stats(): | ||
| return { | ||
| "allocated_bytes.all.current": memory.current, | ||
| "allocated_bytes.all.peak": memory.peak, | ||
| } | ||
|
|
||
| def reset_peak(): | ||
| memory.peak = memory.current | ||
|
|
||
| def await_responses(request_ids): | ||
| record_workspace(decoder_workspace) | ||
| return [] | ||
|
|
||
| def release_resources(): | ||
| memory.temporary_cache = 0 | ||
|
|
||
| monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None) | ||
| monkeypatch.setattr(torch.cuda, "reset_peak_memory_stats", reset_peak) | ||
| monkeypatch.setattr(torch.cuda, "memory_stats", memory_stats) | ||
| monkeypatch.setattr( | ||
| torch.cuda, | ||
| "mem_get_info", | ||
| lambda: (memory.total - memory.current - memory.temporary_cache, memory.total), | ||
| ) | ||
| monkeypatch.setattr(MultimodalParams, "to_device", lambda self, *args, **kwargs: self) | ||
| monkeypatch.setattr("tensorrt_llm._torch.pyexecutor._util.get_sm_version", lambda: 90) | ||
| monkeypatch.setattr("tensorrt_llm._torch.pyexecutor._util.prefer_pinned", lambda: False) | ||
| monkeypatch.setattr( | ||
| "tensorrt_llm._torch.pyexecutor._util.discard_nccl_window_tensor_outputs", | ||
| lambda inputs: nullcontext(), | ||
| ) | ||
|
|
||
| assert creator.try_prepare_estimation() | ||
| memory.temporary_cache = kv_cache_config.max_gpu_total_bytes | ||
| memory.current += 128 << 20 | ||
| kv_manager = SimpleNamespace( | ||
| get_kv_cache_stats=lambda: SimpleNamespace(allocated_bytes=memory.temporary_cache) | ||
| ) | ||
| executor = SimpleNamespace( | ||
| sampler=None, | ||
| enable_iter_perf_stats=False, | ||
| dist=SimpleNamespace(mapping=Mapping(), broadcast=lambda value, root: value), | ||
| resource_manager=SimpleNamespace( | ||
| resource_managers={ResourceManagerType.KV_CACHE_MANAGER: kv_manager} | ||
| ), | ||
| set_gather_responses=Mock(), | ||
| enqueue_requests=Mock(return_value=[1]), | ||
| warmup_scope=nullcontext, | ||
| start_worker=Mock(), | ||
| await_responses=await_responses, | ||
| release_resources=release_resources, | ||
| ) | ||
| creator.configure_kv_cache_capacity(executor) | ||
| assert kv_cache_config.max_gpu_total_bytes == expected_quota | ||
|
|
||
|
|
||
| def _make_creator( | ||
| tokens_per_block, | ||
| dummy_reqs, | ||
|
|
@@ -415,6 +551,7 @@ def _make_creator( | |
| c._net_max_seq_len = 2048 | ||
| c._speculative_config = None | ||
| c._dummy_reqs = dummy_reqs | ||
| c._dummy_encoder_inputs = [] | ||
|
|
||
| c._mapping = Mock(enable_attention_dp=enable_attention_dp, tp_size=tp_size, cp_config={}) | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,13 +15,17 @@ | |
|
|
||
| import pytest | ||
| import torch | ||
| from PIL import Image | ||
| from transformers import AutoProcessor | ||
|
|
||
| from tensorrt_llm import LLM | ||
| from tensorrt_llm._torch.pyexecutor.py_executor_creator import \ | ||
| create_py_executor | ||
| from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManagerType | ||
| from tensorrt_llm.llmapi import (CapacitySchedulerPolicy, DynamicBatchConfig, | ||
| SchedulerConfig) | ||
| from tensorrt_llm.llmapi.llm_args import (CudaGraphConfig, KvCacheConfig, | ||
| SamplingParams, SchedulerConfig) | ||
| from tensorrt_llm.llmapi.llm_args import (CudaGraphConfig, ExecutorMemoryType, | ||
| KvCacheConfig, SleepConfig, | ||
| TorchLlmArgs) | ||
|
|
||
| # isort: off | ||
|
|
@@ -88,6 +92,53 @@ def test_profile_kvcache(): | |
| assert vlm_activation_bytes_with_mm_reqs > vlm_activation_bytes_no_mm_reqs, f"Activation bytes should be higher with mm reqs, but got {vlm_activation_bytes_with_mm_reqs} for mm reqs and {vlm_activation_bytes_no_mm_reqs} without mm reqs" | ||
|
|
||
|
|
||
| def test_multimodal_encoder_profile_survives_release_resume(): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Nice to have] This test passes before and after the change, and also with the |
||
| """Keep multimodal startup buffers usable after releasing their memory tag.""" | ||
| model_path = get_model_path("Qwen3/Qwen3-VL-2B-Instruct") | ||
| processor = AutoProcessor.from_pretrained(model_path) | ||
| prompt = processor.apply_chat_template( | ||
| [{ | ||
| "role": | ||
| "user", | ||
| "content": [{ | ||
| "type": "image" | ||
| }, { | ||
| "type": "text", | ||
| "text": "Describe this image briefly." | ||
| }], | ||
| }], | ||
| tokenize=False, | ||
| add_generation_prompt=True, | ||
| ) | ||
| sampling_params = SamplingParams(max_tokens=8, temperature=0) | ||
| with Image.new("RGB", (32, 32), color=(0, 64, 128)) as image, LLM( | ||
| model_path, | ||
| max_batch_size=1, | ||
| max_num_tokens=128, | ||
| max_seq_len=2048, | ||
| cuda_graph_config=CudaGraphConfig(max_batch_size=1), | ||
| kv_cache_config=KvCacheConfig(max_tokens=4096, | ||
| free_gpu_memory_fraction=0.3, | ||
| enable_block_reuse=False), | ||
| sleep_config=SleepConfig(), | ||
| ) as llm: | ||
| request = {"prompt": prompt, "multi_modal_data": {"image": [image]}} | ||
| expected = llm.generate([request], | ||
| sampling_params)[0].outputs[0].token_ids | ||
| assert expected | ||
| llm.release([ExecutorMemoryType.MODEL_EXTRA]) | ||
| try: | ||
| status = llm.get_memory_status() | ||
| assert status.state == "parked" | ||
| assert status.parked_tags == [ExecutorMemoryType.MODEL_EXTRA] | ||
| finally: | ||
| llm.resume() | ||
| actual = llm.generate([request], | ||
| sampling_params)[0].outputs[0].token_ids | ||
| assert actual == expected | ||
| assert llm.get_memory_status().state == "running" | ||
|
|
||
|
|
||
| def test_pyexecutor_and_kvcache_share_execution_stream(): | ||
| """Test that PyExecutor and KVCacheManager share the same execution_stream. | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Nice to have] The comment that explained the probe design was removed from configure, and the new flow has none. A short note here (why the probe runs before the temporary KV cache and keeps its output resident), at the
max(...)inconfigure_kv_cache_capacity()(the encoder transient and the text-only activations don't coexist; the retained output is already inmodel_bytes), and at thevm_poolslist inallocation_scopewould keep a later edit from moving the probe back or summing the two terms.