SCHED-1041: Replace the GPU propagation lock with a settle-window recheck because bind mounts can be reaped seconds after a passing check.

Author: theyoprst

images/common/scripts/complement_jail.sh

@@ -71,7 +71,7 @@ ensure_nvml_soname() {
     if [ "${how}" = "lock-busy" ] && [ ! -s "${so1}" ]; then
         # The lock holder is still running ldconfig; instead of polling with a guessed delay,
         # wait for the lock to be released — that happens exactly when the holder's ldconfig exits
-        flock --wait 300 etc/complement_jail_ldconfig.lock -c true || echo "[nvml] timed out waiting for the ldconfig lock holder"
+        flock --wait 60 etc/complement_jail_ldconfig.lock -c true || echo "[nvml] timed out waiting for the ldconfig lock holder"
     fi
     if [ ! -s "${so1}" ]; then
         # Grace re-checks: if the symlink becomes usable here, it was created and only this view lagged
@@ -90,7 +90,7 @@ ensure_nvml_soname() {
         # produces correct symlinks no matter what the previous runner saw
         echo "[nvml] ANOMALY: libnvidia-ml.so.1 missing or empty after ldconfig phase (${how}), running ldconfig here (SCHED-1041)"
         dump_nvml_state "before repair ldconfig (${how})"
-        flock --wait 300 etc/complement_jail_ldconfig.lock -c "chroot \"${jaildir}\" /usr/sbin/ldconfig" || echo "[nvml] repair ldconfig failed with exit code $?"
+        flock --wait 60 etc/complement_jail_ldconfig.lock -c "chroot \"${jaildir}\" /usr/sbin/ldconfig" || echo "[nvml] repair ldconfig failed with exit code $?"
         if [ ! -s "${so1}" ]; then
             echo "[nvml] libnvidia-ml.so.1 still missing or empty after repair, exiting so the container restarts (SCHED-1041)"
             dump_nvml_state "final state before exit (${how})"
@@ -122,10 +122,14 @@ gpu_libs_mounted() {
 # Run nvidia-container-cli and make sure its bind mounts survived; re-run when the shared-FS
 # race detached them (a re-run is safe: mounts stack, existing files are not modified), and
 # restart the container if the mounts keep disappearing (SCHED-1041).
+# A check passing right after the CLI is not enough: in the observed reproductions the mounts
+# were reaped ~2s later, when a neighbor worker touched the still-fresh files on the shared
+# jail. So a pass only counts if it survives a short settle window; the jittered sleeps also
+# spread the workers apart so retries do not keep colliding.
 # Uses cap_args/FAKE_LDCONFIG set by the GPU section. Must be called with CWD = ${jaildir}.
 mount_gpu_libs() {
-    local attempt
-    for attempt in 1 2 3; do
+    local attempt backoff
+    for attempt in 1 2 3 4; do
         nvidia-container-cli \
             --user \
             --debug=/dev/stderr \
@@ -137,14 +141,23 @@ mount_gpu_libs() {
             "${cap_args[@]}" \
             "${jaildir}"
         if gpu_libs_mounted; then
-            echo "[nvml] GPU libs mounted (attempt ${attempt})"
-            return 0
+            sleep $((2 + RANDOM % 3))
+            if gpu_libs_mounted; then
+                echo "[nvml] GPU libs mounted and stable (attempt ${attempt})"
+                return 0
+            fi
+            echo "[nvml] ANOMALY: nvidia bind mounts reaped during the settle window, re-running (attempt ${attempt}) (SCHED-1041)"
+        else
+            echo "[nvml] ANOMALY: nvidia bind mounts lost right after nvidia-container-cli, re-running (attempt ${attempt}) (SCHED-1041)"
         fi
-        echo "[nvml] ANOMALY: nvidia bind mounts lost right after nvidia-container-cli, re-running (attempt ${attempt}) (SCHED-1041)"
         dump_nvml_state "after lost mounts, attempt ${attempt}"
-        sleep 2
+        if [ "${attempt}" -lt 4 ]; then
+            # Exponential backoff with jitter: 2-4s, 4-8s, 8-16s
+            backoff=$((2 ** attempt))
+            sleep $((backoff + RANDOM % (backoff + 1)))
+        fi
     done
-    echo "[nvml] nvidia bind mounts still broken after 3 attempts, exiting so the container restarts (SCHED-1041)"
+    echo "[nvml] nvidia bind mounts still broken after 4 attempts, exiting so the container restarts (SCHED-1041)"
     exit 1
 }
 
@@ -230,21 +243,14 @@ pushd "${jaildir}"
 
         echo "[nvml] Jail mount: $(findmnt --target . --output SOURCE,FSTYPE,OPTIONS --noheadings 2>&1)"
 
-        # SCHED-1041: concurrent creation of the driver-lib files on the shared jail makes the kernel silently drop
-        # the earlier worker's bind mounts (remote dentry invalidation on virtiofs),
-        # leaving 0-byte placeholders behind the soname symlinks.
-        # Serialize the propagation across all workers of the cluster: one run takes well under a second, so even
-        # hundreds of workers pass this lock within minutes, and only cold bring-up pays (later runs find the lock free).
-        # The kernel releases the lock automatically if we exit while holding it.
-        exec {gpu_lock_fd}> "etc/complement_jail_gpu_propagate.lock"
-        if ! flock --wait 1800 "${gpu_lock_fd}"; then
-            echo "[nvml] timed out waiting for the GPU propagation lock, exiting so the container restarts"
-            exit 1
-        fi
+        # SCHED-1041: when another worker touches the freshly created driver-lib files on the
+        # shared jail within seconds of their creation, the kernel silently drops this worker's
+        # bind mounts (remote dentry invalidation on virtiofs), leaving 0-byte placeholders
+        # behind the soname symlinks. mount_gpu_libs retries until the mounts survive a settle
+        # window; once the files are more than a few seconds old they are stable (established
+        # clusters never hit this).
         mount_gpu_libs
         touch "etc/gpu_libs_installed.flag"
-        flock --unlock "${gpu_lock_fd}"
-        exec {gpu_lock_fd}>&-
 
         echo "[nvml] /lib symlink: $(readlink lib 2>/dev/null || echo 'NOT a symlink')"
         dump_nvml_state "after nvidia-container-cli"