Skip to content

Commit fb04f50

Browse files
theyoprsttheyoprst
committed
SCHED-1041: Serialize GPU lib propagation across workers and repair broken NVML symlinks, since concurrent nvidia-container-cli runs on the shared jail drop earlier bind mounts.
1 parent f70000e commit fb04f50

1 file changed

Lines changed: 87 additions & 56 deletions

File tree

‎images/common/scripts/complement_jail.sh‎

Lines changed: 87 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -55,44 +55,99 @@ dump_nvml_state() {
5555
echo "[nvml] === end NVML jail state: ${1} ==="
5656
}
5757

58-
# Verify the NVIDIA soname symlink exists after the ldconfig phase and log the SCHED-1041 anomaly otherwise.
59-
# ${1} says what happened on this worker:
58+
# Make sure the NVIDIA soname symlink exists AND points at real content after the ldconfig phase:
59+
# repair by running ldconfig here if not, and restart the container when the jail still has no
60+
# working NVML (SCHED-1041). ${1} says what happened on this worker:
6061
# - "ran" (ldconfig executed here),
6162
# - "lock-busy" (another worker holds the ldconfig lock),
6263
# - "skipped" (this worker decided not to run it), or
6364
# - "failed".
64-
# Kept generic so main's probe-based ldconfig block (PR #2621) can call it with the same statuses ("skipped" when
65-
# the probe reports the cache as up to date — an anomaly then means the probe was fooled by a broken view of the jail).
65+
# Kept generic so main's probe-based ldconfig block (PR #2621) can call it with the same statuses.
66+
# -s follows the symlink, so a dangling link or an empty placeholder target counts as broken.
6667
# Must be called with CWD = ${jaildir}.
67-
verify_nvml_soname() {
68+
ensure_nvml_soname() {
6869
local how=$1
6970
local so1="usr/lib/${ALT_ARCH}-linux-gnu/libnvidia-ml.so.1"
70-
if [ "${how}" = "lock-busy" ] && [ ! -e "${so1}" ]; then
71+
if [ "${how}" = "lock-busy" ] && [ ! -s "${so1}" ]; then
7172
# The lock holder is still running ldconfig; instead of polling with a guessed delay,
7273
# wait for the lock to be released — that happens exactly when the holder's ldconfig exits
7374
flock --wait 300 etc/complement_jail_ldconfig.lock -c true || echo "[nvml] timed out waiting for the ldconfig lock holder"
7475
fi
75-
if [ ! -e "${so1}" ]; then
76-
# Grace re-checks before declaring the anomaly: if the symlink shows up here, it
77-
# was created and only this view lagged
76+
if [ ! -s "${so1}" ]; then
77+
# Grace re-checks: if the symlink becomes usable here, it was created and only this view lagged
7878
local waited=0
7979
for delay in 1 5; do
8080
sleep "${delay}"
8181
waited=$((waited + delay))
82-
if [ -e "${so1}" ]; then
83-
echo "[nvml] libnvidia-ml.so.1 appeared after ${waited}s extra wait (view lag)"
82+
if [ -s "${so1}" ]; then
83+
echo "[nvml] libnvidia-ml.so.1 usable after ${waited}s extra wait (view lag)"
8484
break
8585
fi
8686
done
8787
fi
88-
if [ -e "${so1}" ]; then
89-
echo "[nvml] libnvidia-ml.so.1 present after ldconfig phase (${how})"
88+
if [ ! -s "${so1}" ]; then
89+
# Repair: this worker's own view is verified healthy (mount_gpu_libs), so its ldconfig
90+
# produces correct symlinks no matter what the previous runner saw
91+
echo "[nvml] ANOMALY: libnvidia-ml.so.1 missing or empty after ldconfig phase (${how}), running ldconfig here (SCHED-1041)"
92+
dump_nvml_state "before repair ldconfig (${how})"
93+
flock --wait 300 etc/complement_jail_ldconfig.lock -c "chroot \"${jaildir}\" /usr/sbin/ldconfig" || echo "[nvml] repair ldconfig failed with exit code $?"
94+
if [ ! -s "${so1}" ]; then
95+
echo "[nvml] libnvidia-ml.so.1 still missing or empty after repair, exiting so the container restarts (SCHED-1041)"
96+
dump_nvml_state "final state before exit (${how})"
97+
exit 1
98+
fi
99+
echo "[nvml] libnvidia-ml.so.1 repaired by local ldconfig"
90100
else
91-
echo "[nvml] ANOMALY: libnvidia-ml.so.1 absent after ldconfig phase (${how}) (SCHED-1041)"
101+
echo "[nvml] libnvidia-ml.so.1 usable after ldconfig phase (${how})"
92102
fi
93103
dump_nvml_state "after ldconfig phase (${how})"
94104
}
95105

106+
# Success when this worker's NVML bind mount is attached and the versioned lib has real
107+
# content through the path view. Both go away together when the kernel drops the mounts
108+
# on remote dentry invalidation (SCHED-1041). Must be called with CWD = ${jaildir}.
109+
gpu_libs_mounted() {
110+
if [ "$(grep -cF "${jaildir}/usr/lib/${ALT_ARCH}-linux-gnu/libnvidia-ml" /proc/self/mountinfo)" -lt 1 ]; then
111+
return 1
112+
fi
113+
local lib
114+
for lib in "usr/lib/${ALT_ARCH}-linux-gnu/"libnvidia-ml.so.*.*; do
115+
if [ -s "${lib}" ]; then
116+
return 0
117+
fi
118+
done
119+
return 1
120+
}
121+
122+
# Run nvidia-container-cli and make sure its bind mounts survived; re-run when the shared-FS
123+
# race detached them (a re-run is safe: mounts stack, existing files are not modified), and
124+
# restart the container if the mounts keep disappearing (SCHED-1041).
125+
# Uses cap_args/FAKE_LDCONFIG set by the GPU section. Must be called with CWD = ${jaildir}.
126+
mount_gpu_libs() {
127+
local attempt
128+
for attempt in 1 2 3; do
129+
nvidia-container-cli \
130+
--user \
131+
--debug=/dev/stderr \
132+
--no-pivot \
133+
configure \
134+
--no-cgroups \
135+
--ldconfig=$FAKE_LDCONFIG \
136+
--device=all \
137+
"${cap_args[@]}" \
138+
"${jaildir}"
139+
if gpu_libs_mounted; then
140+
echo "[nvml] GPU libs mounted (attempt ${attempt})"
141+
return 0
142+
fi
143+
echo "[nvml] ANOMALY: nvidia bind mounts lost right after nvidia-container-cli, re-running (attempt ${attempt}) (SCHED-1041)"
144+
dump_nvml_state "after lost mounts, attempt ${attempt}"
145+
sleep 2
146+
done
147+
echo "[nvml] nvidia bind mounts still broken after 3 attempts, exiting so the container restarts (SCHED-1041)"
148+
exit 1
149+
}
150+
96151
pushd "${jaildir}"
97152
echo "Bind-mount virtual filesystems"
98153
mount -t proc /proc proc/
@@ -175,48 +230,24 @@ pushd "${jaildir}"
175230

176231
echo "[nvml] Jail mount: $(findmnt --target . --output SOURCE,FSTYPE,OPTIONS --noheadings 2>&1)"
177232

178-
nvidia-container-cli \
179-
--user \
180-
--debug=/dev/stderr \
181-
--no-pivot \
182-
configure \
183-
--no-cgroups \
184-
--ldconfig=$FAKE_LDCONFIG \
185-
--device=all \
186-
"${cap_args[@]}" \
187-
"${jaildir}"
233+
# SCHED-1041: concurrent creation of the driver-lib files on the shared jail makes the kernel silently drop
234+
# the earlier worker's bind mounts (remote dentry invalidation on virtiofs),
235+
# leaving 0-byte placeholders behind the soname symlinks.
236+
# Serialize the propagation across all workers of the cluster: one run takes well under a second, so even
237+
# hundreds of workers pass this lock within minutes, and only cold bring-up pays (later runs find the lock free).
238+
# The kernel releases the lock automatically if we exit while holding it.
239+
exec {gpu_lock_fd}> "etc/complement_jail_gpu_propagate.lock"
240+
if ! flock --wait 1800 "${gpu_lock_fd}"; then
241+
echo "[nvml] timed out waiting for the GPU propagation lock, exiting so the container restarts"
242+
exit 1
243+
fi
244+
mount_gpu_libs
245+
touch "etc/gpu_libs_installed.flag"
246+
flock --unlock "${gpu_lock_fd}"
247+
exec {gpu_lock_fd}>&-
188248

189249
echo "[nvml] /lib symlink: $(readlink lib 2>/dev/null || echo 'NOT a symlink')"
190250
dump_nvml_state "after nvidia-container-cli"
191-
192-
# Local-coherence probe: a file created through this view must be immediately visible;
193-
# if it is not, the whole directory view is incoherent, not just the freshly created bind mounts
194-
probe="usr/lib/${ALT_ARCH}-linux-gnu/.soperator-jail-probe-$(hostname)-$$"
195-
touch "${probe}" || echo "[nvml] probe touch FAILED"
196-
if [ -e "${probe}" ]; then
197-
echo "[nvml] probe file visible right after touch: yes"
198-
else
199-
echo "[nvml] probe file visible right after touch: NO (local writes invisible through this view)"
200-
fi
201-
rm -f "${probe}" || true
202-
203-
# If the versioned lib only appears after a delay, the staleness is transient and a retry-based fix is viable;
204-
# if it never appears, the view stays broken
205-
if ! compgen -G "usr/lib/${ALT_ARCH}-linux-gnu/libnvidia-ml.so.*.*" > /dev/null; then
206-
waited=0
207-
for delay in 1 5; do
208-
sleep "${delay}"
209-
waited=$((waited + delay))
210-
if compgen -G "usr/lib/${ALT_ARCH}-linux-gnu/libnvidia-ml.so.*.*" > /dev/null; then
211-
echo "[nvml] versioned libnvidia-ml appeared after ${waited}s"
212-
break
213-
fi
214-
echo "[nvml] versioned libnvidia-ml still not visible after ${waited}s"
215-
done
216-
dump_nvml_state "after visibility wait"
217-
fi
218-
219-
touch "etc/gpu_libs_installed.flag"
220251
fi
221252

222253
echo "Bind-mount slurm client"
@@ -301,9 +332,9 @@ pushd "${jaildir}"
301332
fi
302333
if [ "$SLURM_CLUSTER_WITH_GPU" = "true" ]; then
303334
case "$ldconfig_rc" in
304-
0) verify_nvml_soname "ran" ;;
305-
1) verify_nvml_soname "lock-busy" ;;
306-
*) verify_nvml_soname "failed" ;;
335+
0) ensure_nvml_soname "ran" ;;
336+
1) ensure_nvml_soname "lock-busy" ;;
337+
*) ensure_nvml_soname "failed" ;;
307338
esac
308339
fi
309340
fi

0 commit comments

Comments
 (0)