Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 13 additions & 24 deletions tensorrt_llm/_torch/peft/lora/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,8 @@ def load_from_hf(
for model_dir in new_model_dirs:
with open(f"{model_dir}/adapter_config.json", "r") as f:
config = json.load(f)
if config.get("use_dora", False):
raise NotImplementedError("DoRA adapters are not supported")
lora_hf_configs.append(config)

self.lora_target_modules = model_config.lora_target_modules
Expand Down Expand Up @@ -686,6 +688,15 @@ def load_from_model_dir(uid, model_dir, hf_config):
owner=getattr(model_config, "owner", DecoderLoraOwner()),
hf_modules_to_trtllm_modules=hf_modules_to_trtllm_modules,
)
for layer_weights in all_weights.values():
for module_weights in layer_weights.values():
weights_to_check = (
module_weights.values()
if _is_moe_module_weights(module_weights)
else [module_weights]
)
if any("magnitude" in weights for weights in weights_to_check):
raise NotImplementedError("DoRA adapters are not supported")
rank = int(hf_config["r"])
rs_lora = bool(hf_config.get("use_rslora", False))
model_dtype = str_dtype_to_torch(model_config.dtype)
Expand Down Expand Up @@ -781,11 +792,6 @@ def uses_native_fp8(module_weights):

t_in = torch.stack(t_in_list)
t_out = torch.stack(t_out_list)
for weights in module_weights.values():
if "mag" in weights:
# TODO(oargov): this might work, but I had no MoE DoRA models to test
raise ValueError("DoRA with MoE is not supported")
t_mag = None
else:
# Not MoE - validate required matrices are present
_check_lora_in_out(
Expand All @@ -797,9 +803,6 @@ def uses_native_fp8(module_weights):

t_in = module_weights["in"]
t_out = module_weights["out"]
t_mag = module_weights.get("magnitude", None)

is_dora = t_mag is not None
rank_dim = 1 if has_expert_indices else 0
t_out = prepare_fused_lora_modules_for_tp(lora_module, t_out, rank_dim)

Expand All @@ -820,15 +823,9 @@ def uses_native_fp8(module_weights):
layer_idx=layer_idx,
lora_module=lora_module,
)
if is_dora:
raise NotImplementedError(
"DoRA is not supported with FP8 LoRA weights on SM90/SM100"
)

t_in = t_in.cuda().contiguous()
t_out = t_out.cuda().contiguous()
if is_dora and t_mag is not None:
t_mag = t_mag.cuda().contiguous()

if rs_lora:
scale = float(hf_config["lora_alpha"]) / np.sqrt(effective_rank)
Expand All @@ -849,33 +846,25 @@ def uses_native_fp8(module_weights):
t_in = t_in.to(model_dtype)
t_out = t_out.to(model_dtype)
t_out = t_out * scale
if is_dora and t_mag is not None:
t_mag = t_mag.to(model_dtype)

uid_to_low_ranks[layer_idx][lora_module] = effective_rank
if self._retain_device_tensors:
lora_weights_pointers[layer_idx][lora_module] = [
t_in.data_ptr(),
t_out.data_ptr(),
t_mag.data_ptr() if (is_dora and t_mag is not None) else 0,
0,
]
retained_lora_weights.append(t_in)
retained_lora_weights.append(t_out)
if is_dora and t_mag is not None:
retained_lora_weights.append(t_mag)

t_in_cpu = t_in.flatten().cpu()
t_out_cpu = t_out.flatten().cpu()
weights_to_concat = [t_in_cpu, t_out_cpu]

if is_dora and t_mag is not None:
t_mag_cpu = t_mag.flatten().cpu()
weights_to_concat.append(t_mag_cpu)

cpp_lora_weights.append(torch.cat(weights_to_concat))
cpp_lora_config.append(
torch.tensor(
[self.LORA_MODULE_IDS[lora_module], layer_idx, effective_rank, is_dora],
[self.LORA_MODULE_IDS[lora_module], layer_idx, effective_rank, False],
dtype=torch.int32,
)
)
Expand Down
92 changes: 63 additions & 29 deletions tests/unittest/others/test_lora_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest
import torch
from safetensors.torch import load_file, save_file

Expand Down Expand Up @@ -142,6 +143,68 @@ def _create_dummy_hf_moe_lora_adapter(
save_file(weights, str(adapter_dir / "adapter_model.safetensors"))


@pytest.mark.cpu_only
@pytest.mark.parametrize(
"dtype,is_moe,use_dora",
[(torch.float16, False, True)]
+ [
(dtype, is_moe, use_dora)
for dtype in (torch.float16, torch.bfloat16, torch.float8_e4m3fn)
for is_moe in (False, True)
for use_dora in (False, None)
],
)
def test_dora_is_rejected(
tmp_path: Path, dtype: torch.dtype, is_moe: bool, use_dora: bool | None
) -> None:
"""Reject DoRA metadata or magnitude weights before touching CUDA or registering an adapter."""
if is_moe:
_create_dummy_hf_moe_lora_adapter(tmp_path, dtype=dtype)
model_config = MockModelConfig(
lora_target_modules=["moe_h_to_4h"],
trtllm_modules_to_hf_modules={"moe_h_to_4h": "mlp.experts.gate_proj"},
)
weights_path = tmp_path / "adapter_model.safetensors"
weights = load_file(str(weights_path))
prefix = "base_model.model.model.layers.0.mlp.experts.0.gate_proj"
weights[f"{prefix}.lora_magnitude_vector"] = torch.ones(128)
save_file(weights, str(weights_path))
else:
_create_dummy_hf_lora_adapter(tmp_path, dtype=dtype, use_dora=True)
model_config = MockModelConfig()

config_path = tmp_path / "adapter_config.json"
config = json.loads(config_path.read_text())
if use_dora is None:
config.pop("use_dora", None)
else:
config["use_dora"] = use_dora
config_path.write_text(json.dumps(config))
if use_dora:
# The configuration alone must suffice, even without checkpoint weights.
(tmp_path / "adapter_model.safetensors").unlink()

manager = LoraManager(
mapping=Mapping(world_size=1, rank=0, tp_size=1),
model_config=model_config,
cpp_peft_cache_manager=MagicMock(),
)
with (
patch.object(torch.Tensor, "cuda") as transfer,
patch.object(torch.cuda, "get_device_capability") as capability,
pytest.raises(NotImplementedError, match="DoRA adapters are not supported"),
):
manager.load_from_hf([str(tmp_path)], model_config, uids=["dora"])

transfer.assert_not_called()
capability.assert_not_called()
assert manager.num_lora_adapters == 0
assert not manager.cpp_lora_weights
assert not manager.cpp_lora_config
assert not manager.lora_weights
assert not manager.lora_weights_pointers_list


@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
class TestLoraManagerRetainDeviceTensors(unittest.TestCase):
"""Tests for the _retain_device_tensors flag that prevents GPU memory leaks."""
Expand Down Expand Up @@ -259,35 +322,6 @@ def test_hf_loader_reports_expert_only_fp8_adapter_dtype(self):

self.assertEqual(loader.get_lora_dtype(), torch.float8_e4m3fn)

def test_fp8_dora_is_rejected(self):
model_config = MockModelConfig()
manager = LoraManager(
mapping=Mapping(world_size=1, rank=0, tp_size=1),
model_config=model_config,
cpp_peft_cache_manager=MagicMock(),
)

with tempfile.TemporaryDirectory() as tmpdir:
adapter_dir = Path(tmpdir) / "adapter"
adapter_dir.mkdir()
_create_dummy_hf_lora_adapter(
adapter_dir,
rank=16,
num_layers=1,
dtype=torch.float8_e4m3fn,
use_dora=True,
)

with self.assertRaisesRegex(
NotImplementedError,
"DoRA is not supported with FP8 LoRA weights on SM90/SM100",
):
manager.load_from_hf(
model_dirs=[str(adapter_dir)],
model_config=model_config,
uids=["fp8-dora"],
)

def test_fp8_moe_weights_remain_fp8(self):
model_config = MockModelConfig(
lora_target_modules=["moe_h_to_4h"],
Expand Down
Loading