SCHED-1290 Run more checks in one slurm job for active checks

ansible/roles/nc-health-checker/defaults/main.yml

@@ -1,3 +1,3 @@
 ---
 
-nc_health_checker_version: 1.1.15-b3af5269
+nc_health_checker_version: 1.1.18-09154e5d

helm/slurm-cluster/slurm_scripts/gpu_health_check.py

@@ -9,7 +9,7 @@
 
 
 # Open the directory from which the checks should be run
-def chdir_into_tmp():
+def chdir_into_tmp() -> None:
     try:
         os.chdir("/tmp")
     except Exception as e:
@@ -23,7 +23,7 @@ def print_hc_result(
     hc_exitcode: int = -1,
     hc_stdout: str = None,
     hc_stderr: str = None,
-):
+) -> None:
     print(res_desc)
     print(f"Health checker exit code: {hc_exitcode}")
     print("Health checker stdout:")
@@ -77,7 +77,7 @@ def get_hc_result(proc: subprocess.CompletedProcess) -> HealthCheckerResult:
         return res
 
 
-def ensure_output_dir(path_str: str):
+def ensure_output_dir(path_str: str) -> None:
     path = pathlib.Path(path_str)
 
     old_umask = os.umask(0)
@@ -100,15 +100,7 @@ def ensure_output_dir(path_str: str):
     # Change into /tmp before running health-checker
     chdir_into_tmp()
 
-    # Define tests to run
-    tests: str = ""
-    if CHECKS_CONTEXT == "prolog":
-        tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dmesg,ib_link"
-    elif CHECKS_CONTEXT == "epilog":
-        tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dcgmi_diag_r1,dmesg,ib_link"
-    elif CHECKS_CONTEXT == "hc_program":
-        tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dmesg,ib_link"
-    else:
+    if CHECKS_CONTEXT not in ("prolog", "epilog", "hc_program"):
         print(f"Unknown context '{CHECKS_CONTEXT}'")
         sys.exit(0)
 
@@ -122,9 +114,8 @@ def ensure_output_dir(path_str: str):
     # Run Nebius GPU health-checker
     cmd = [
         "health-checker", "run",
-        "-e", "soperator",
+        "-e", f"soperator_{CHECKS_CONTEXT}",
         "-p", CHECKS_PLATFORM_TAG,
-        "-n", tests,
         "-f", "json-partial",
         "--tests-stdout-path", output_dir,
         "--log-level", "info",

helm/soperator-activechecks/scripts/dcgmi-diag-r2.sh

@@ -23,7 +23,7 @@ fi
 
 echo "Platform found: $platform"
 echo "Running dcgmi_diag_r2 check on $(hostname)..."
-HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator -p $platform -n dcgmi_diag_r2 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
+HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -i dcgmi_diag_r2 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
 
 echo "Health checker output:"
 echo "$HC_OUTPUT"

helm/soperator-activechecks/scripts/dcgmi-diag-r3.sh

@@ -23,7 +23,7 @@ fi
 
 echo "Platform found: $platform"
 echo "Running dcgmi_diag_r3 check on $(hostname)..."
-HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -n dcgmi_diag_r3 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
+HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -i dcgmi_diag_r3 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
 
 echo "Health checker output:"
 echo "$HC_OUTPUT"

helm/soperator-activechecks/scripts/all-reduce-perf-nccl-with-ib.sh → helm/soperator-activechecks/scripts/gpu-checks.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#SBATCH --deadline="now+4hours"
+#SBATCH --deadline="now+12hours"
 #SBATCH --time=30:00
 #SBATCH --exclusive
 #SBATCH --mem=0
@@ -22,8 +22,8 @@ else
 fi
 
 echo "Platform found: $platform"
-echo "Running all_reduce_perf_nccl check on $(hostname)..."
-HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator -p $platform -n all_reduce_with_ib -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout --log-level info")
+echo "Running GPU checks on $(hostname)..."
+HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator_active_checks -p $platform -x '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
 
 echo "Health checker output:"
 echo "$HC_OUTPUT"

helm/soperator-activechecks/scripts/ib-gpu-perf.sh

@@ -26,7 +26,7 @@ echo "Running ib_gpu_perf check on $(hostname)..."
 HC_OUTPUT_DIR="/opt/soperator-outputs/health_checker_cmd_stdout"
 HC_OUTPUT=$(srun --container-image={{ include "activecheck.image.pyxis" . }} \
   --container-mounts=$(which health-checker):/usr/local/bin/health-checker,$HC_OUTPUT_DIR:$HC_OUTPUT_DIR --cpu-bind=cores \
-  bash -l -c "health-checker run -e soperator -p $platform -n '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
+  bash -l -c "health-checker run -e soperator_active_checks -p $platform -i '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout")
 
 echo "Health checker output:"
 echo "$HC_OUTPUT"

helm/soperator-activechecks/tests/reactions_conflict_test.yaml

@@ -41,49 +41,49 @@ tests:
     asserts:
       - notFailedTemplate: {}
 
-  - it: gpu-fryer should fail if both commentSlurmNode and drainSlurmNode are set under failureReactions
+  - it: gpu-checks should fail if both commentSlurmNode and drainSlurmNode are set under failureReactions
     template: templates/active-checks.yaml
     set:
       checks:
-        gpu-fryer:
+        gpu-checks:
           failureReactions:
             commentSlurmNode:
               commentPrefix: "[node_problem]"
             drainSlurmNode:
               drainReasonPrefix: "[node_problem]"
     asserts:
       - failedTemplate:
-          errorMessage: "checks.gpu-fryer.failureReactions: cannot set both commentSlurmNode and drainSlurmNode simultaneously"
+          errorMessage: "checks.gpu-checks.failureReactions: cannot set both commentSlurmNode and drainSlurmNode simultaneously"
 
-  - it: gpu-fryer should render when only commentSlurmNode is set
+  - it: gpu-checks should render when only commentSlurmNode is set
     template: templates/active-checks.yaml
     set:
       checks:
-        gpu-fryer:
+        gpu-checks:
           failureReactions:
             commentSlurmNode:
               commentPrefix: "[node_problem]"
             drainSlurmNode: null
     asserts:
       - notFailedTemplate: {}
 
-  - it: gpu-fryer should render when only drainSlurmNode is set
+  - it: gpu-checks should render when only drainSlurmNode is set
     template: templates/active-checks.yaml
     set:
       checks:
-        gpu-fryer:
+        gpu-checks:
           failureReactions:
             commentSlurmNode: null
             drainSlurmNode:
               drainReasonPrefix: "[node_problem]"
     asserts:
       - notFailedTemplate: {}
 
-  - it: gpu-fryer should render when both reaction nodes are explicitly null
+  - it: gpu-checks should render when both reaction nodes are explicitly null
     template: templates/active-checks.yaml
     set:
       checks:
-        gpu-fryer:
+        gpu-checks:
           failureReactions:
             commentSlurmNode: null
             drainSlurmNode: null

helm/soperator-activechecks/tests/slurmjob_gpu_test.yaml

@@ -5,7 +5,7 @@ tests:
   - it: should render default SBATCH_GPUS_PER_NODE for GPU slurm jobs
     documentSelector:
       path: metadata.name
-      value: soperator-cuda-samples
+      value: soperator-gpu-checks
     asserts:
       - contains:
           path: spec.slurmJobSpec.jobContainer.env
@@ -16,7 +16,7 @@ tests:
   - it: should render overridden SBATCH_GPUS_PER_NODE for GPU slurm jobs
     documentSelector:
       path: metadata.name
-      value: soperator-cuda-samples
+      value: soperator-gpu-checks
     set:
       slurmJob:
         gpusPerNode: "4"

helm/soperator-activechecks/tests/slurmjob_spec_test.yaml

@@ -3,10 +3,10 @@ templates:
   - templates/active-checks.yaml
 
 tests:
-  - it: should render cuda-samples active check
+  - it: should render gpu-checks active check
     documentSelector:
       path: metadata.name
-      value: soperator-cuda-samples
+      value: soperator-gpu-checks
     asserts:
       - isKind:
           of: ActiveCheck
@@ -21,10 +21,10 @@ tests:
           value: false
       - contains:
           path: spec.dependsOn
-          content: soperator-prepull-container-image
+          content: soperator-manage-jail-state
       - matchRegex:
           path: spec.slurmJobSpec.sbatchScript
           pattern: 'health-checker run'
       - matchRegex:
           path: spec.slurmJobSpec.sbatchScript
-          pattern: 'Running cuda samples check'
+          pattern: 'Running GPU checks'