Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 38 additions & 19 deletions jenkins/TensorRT_LLM_PLC.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,26 @@ def createKubernetesPodConfig()
tty: true
resources:
requests:
cpu: '8'
memory: 32Gi
cpu: '1'
memory: 5Gi
ephemeral-storage: 200Gi
limits:
cpu: '1'
memory: 5Gi
ephemeral-storage: 200Gi
imagePullPolicy: Always
- name: sonarqube
image: ${image}
command: ['cat']
tty: true
resources:
requests:
cpu: '1'
memory: 12Gi
ephemeral-storage: 200Gi
limits:
cpu: '8'
memory: 32Gi
cpu: '1'
memory: 12Gi
ephemeral-storage: 200Gi
imagePullPolicy: Always
- name: jnlp
Expand All @@ -58,24 +72,24 @@ def createKubernetesPodConfig()
resources:
requests:
cpu: '2'
memory: 5Gi
memory: 4Gi
ephemeral-storage: 25Gi
limits:
cpu: '2'
memory: 5Gi
memory: 4Gi
ephemeral-storage: 25Gi
- name: pulse-malware-scanner
image: gitlab-master.nvidia.com:5005/pstooling/pulse-group/pulse-malware-scanner-cli:stable
command: ['sleep', '${scannerSleepSeconds}']
tty: true
resources:
requests:
cpu: '16'
memory: 64Gi
cpu: '1'
memory: 2Gi
ephemeral-storage: 200Gi
limits:
cpu: '16'
memory: 64Gi
cpu: '1'
memory: 2Gi
ephemeral-storage: 200Gi
imagePullPolicy: Always
securityContext:
Expand All @@ -87,12 +101,12 @@ def createKubernetesPodConfig()
tty: true
resources:
requests:
cpu: '16'
memory: 64Gi
cpu: '2'
memory: 5Gi
ephemeral-storage: 200Gi
limits:
cpu: '16'
memory: 64Gi
cpu: '2'
memory: 5Gi
ephemeral-storage: 200Gi
imagePullPolicy: Always
securityContext:
Expand All @@ -104,12 +118,12 @@ def createKubernetesPodConfig()
tty: true
resources:
requests:
cpu: '16'
memory: 64Gi
cpu: '1'
memory: 5Gi
ephemeral-storage: 200Gi
limits:
cpu: '16'
memory: 64Gi
cpu: '1'
memory: 5Gi
ephemeral-storage: 200Gi
imagePullPolicy: Always
securityContext:
Expand Down Expand Up @@ -283,7 +297,12 @@ def generateLockFiles(llmRepo, ref)

def sonarScan()
{
container("cpu") {
container("sonarqube") {
// This container is a bare ubuntu image; the tools installed by installTools()
// live in the "cpu" container's filesystem, not here.
sh "apt update"
sh "apt install -y --no-install-recommends git wget unzip openjdk-17-jre-headless ca-certificates"
sh "git config --global --add safe.directory ${env.WORKSPACE}"
def sonarScannerCliVer = "8.0.0.6341"
// Expected SHA-256 of the scanner zip; update it together with sonarScannerCliVer.
def sonarScannerCliSha256 = "c55773294e951dcba23ebe7084616f33721b0ce0d199401c7e07d3a6cdfafdb1"
Expand Down
99 changes: 87 additions & 12 deletions scripts/generate_lock_file.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -56,7 +56,7 @@
}

target_env = {
"python_version": 310,
"python_version": 312,
"platform_system": "",
"platform_machine": "x86_64",
"sys_platform": "linux",
Expand Down Expand Up @@ -89,6 +89,83 @@ def get_project_info(path: str):
return {"name": name, "version": version}


def _normalize_name(name):
return re.sub(r"[-_.]+", "-", name).lower()


def _find_unresolvable_spec(output, specs):
"""Return the index of the spec that Poetry blames for a failed resolution.

Poetry ends a failed resolution with
"So, because <project> depends on <pkg> (<constraint>), version solving
failed." (or "... on both <a> (..) and <b> (..)", or "Could not find a
matching version of package <pkg>"). When several packages are named, the
one listed last in `specs` is blamed, so that earlier entries win, as when
adding one by one. Returns None if no listed package can be identified.
"""
names = set()
match = re.search(
r"So, because .*? depends on (.*?), version solving failed", output,
re.S)
if match:
names.update(
_normalize_name(n) for n in re.findall(
r"([A-Za-z0-9][A-Za-z0-9_.\-]*)(?:\[[^\]]*\])? \(",
match.group(1)))
match = re.search(r"Could not find a matching version of package (\S+)",
output)
if match:
names.add(_normalize_name(match.group(1)))
indexes = []
for i, spec in enumerate(specs):
match = re.match(r"[A-Za-z0-9][A-Za-z0-9_.\-]*", spec)
if match and _normalize_name(match.group(0)) in names:
indexes.append(i)
return max(indexes) if indexes else None


def _is_resolution_failure(output):
"""Whether Poetry's output reports a dependency-resolution failure.

Only such failures justify skipping a spec. Anything else (network or index
errors, auth problems, a Poetry crash) says nothing about the spec itself.
"""
return bool(
re.search(
r"version solving failed|"
r"Could not find a matching version of package", output))


def poetry_add(specs, cwd):
specs = list(specs)
while specs:
result = subprocess.run(["poetry", "add", "--lock", *specs],
cwd=cwd,
capture_output=True,
text=True)
print(result.stdout, end="")
print(result.stderr, end="", file=sys.stderr)
if result.returncode == 0:
return
output = result.stdout + result.stderr
if not _is_resolution_failure(output):
raise RuntimeError(
f"'poetry add' failed in {cwd} (exit code {result.returncode}) "
"for a reason other than dependency resolution; refusing to "
"produce an incomplete lock file")
culprit = _find_unresolvable_spec(output, specs)
if culprit is None:
if len(specs) == 1:
print(f"Skipping {specs[0]}: dependency resolution failed")
return
mid = len(specs) // 2
poetry_add(specs[:mid], cwd)
poetry_add(specs[mid:], cwd)
return
print(f"Skipping {specs[culprit]}: dependency resolution failed")
del specs[culprit]


def generate_metadata_json():
try:
commit_hash = subprocess.check_output(["git", "rev-parse", "HEAD"],
Expand Down Expand Up @@ -158,11 +235,10 @@ def generate_metadata_json():
shell=True,
capture_output=True,
text=True).stdout
for constraint in project_constraints.get(
str(Path(file_path).relative_to(curr_path)), []):
subprocess.run(f"poetry add '{constraint}'",
shell=True,
cwd=save_path)
# Constraints go first so they steer the resolver for the whole batch.
specs = list(
project_constraints.get(str(Path(file_path).relative_to(curr_path)),
[]))
packages = output.split('\n')

if packages[-1] == '': # last entry is newline
Expand All @@ -174,7 +250,7 @@ def generate_metadata_json():
if not package or \
package.startswith('#') or \
'-f' in package or \
package.startswith('--'):
package.startswith('-'):
continue

curr_env = None
Expand All @@ -201,7 +277,6 @@ def generate_metadata_json():
package_name = data[0].replace(" ", "")
if (not package_name.startswith("tensorrt_llm")
and not package_name.startswith("3rdparty")):
poetry_cmd = f"poetry add '{package_name}'"
if package_name in url_mapping:
poetry_cmd = f"poetry add '{url_mapping[package_name]}'"
subprocess.run(poetry_cmd, shell=True, cwd=save_path)
specs.append(url_mapping.get(package_name, package_name))

poetry_add(list(dict.fromkeys(specs)), save_path)
Loading