diff --git a/ansible/roles/nc-health-checker/defaults/main.yml b/ansible/roles/nc-health-checker/defaults/main.yml index 1e4f636de..0762578e3 100644 --- a/ansible/roles/nc-health-checker/defaults/main.yml +++ b/ansible/roles/nc-health-checker/defaults/main.yml @@ -1,3 +1,3 @@ --- -nc_health_checker_version: 1.1.15-b3af5269 +nc_health_checker_version: 1.1.18-09154e5d diff --git a/helm/slurm-cluster/slurm_scripts/gpu_health_check.py b/helm/slurm-cluster/slurm_scripts/gpu_health_check.py index f6a63ce6b..cbe5b6c0d 100644 --- a/helm/slurm-cluster/slurm_scripts/gpu_health_check.py +++ b/helm/slurm-cluster/slurm_scripts/gpu_health_check.py @@ -9,7 +9,7 @@ # Open the directory from which the checks should be run -def chdir_into_tmp(): +def chdir_into_tmp() -> None: try: os.chdir("/tmp") except Exception as e: @@ -23,7 +23,7 @@ def print_hc_result( hc_exitcode: int = -1, hc_stdout: str = None, hc_stderr: str = None, -): +) -> None: print(res_desc) print(f"Health checker exit code: {hc_exitcode}") print("Health checker stdout:") @@ -77,7 +77,7 @@ def get_hc_result(proc: subprocess.CompletedProcess) -> HealthCheckerResult: return res -def ensure_output_dir(path_str: str): +def ensure_output_dir(path_str: str) -> None: path = pathlib.Path(path_str) old_umask = os.umask(0) @@ -100,15 +100,7 @@ def ensure_output_dir(path_str: str): # Change into /tmp before running health-checker chdir_into_tmp() - # Define tests to run - tests: str = "" - if CHECKS_CONTEXT == "prolog": - tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dmesg,ib_link" - elif CHECKS_CONTEXT == "epilog": - tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dcgmi_diag_r1,dmesg,ib_link" - elif CHECKS_CONTEXT == "hc_program": - tests = "module,nvidia_smi,nvidia_smi_nvlink,nvidia_smi_topo,dmesg,ib_link" - else: + if CHECKS_CONTEXT not in ("prolog", "epilog", "hc_program"): print(f"Unknown context '{CHECKS_CONTEXT}'") sys.exit(0) @@ -122,9 +114,8 @@ def ensure_output_dir(path_str: str): # Run Nebius GPU health-checker cmd = [ "health-checker", "run", - "-e", "soperator", + "-e", f"soperator_{CHECKS_CONTEXT}", "-p", CHECKS_PLATFORM_TAG, - "-n", tests, "-f", "json-partial", "--tests-stdout-path", output_dir, "--log-level", "info", diff --git a/helm/soperator-activechecks/scripts/all-reduce-perf-nccl-without-ib.sh b/helm/soperator-activechecks/scripts/all-reduce-perf-nccl-without-ib.sh deleted file mode 100644 index e8b514c89..000000000 --- a/helm/soperator-activechecks/scripts/all-reduce-perf-nccl-without-ib.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -#SBATCH --deadline="now+4hours" -#SBATCH --time=20:00 -#SBATCH --exclusive -#SBATCH --mem=0 - -platform="" -gpus_on_node=$(nvidia-smi --query-gpu=name --format=csv,noheader | sort | uniq -c) -if [[ "${gpus_on_node}" == *"8 NVIDIA H100"* ]]; then - platform="8xH100" -elif [[ "${gpus_on_node}" == *"8 NVIDIA H200"* ]]; then - platform="8xH200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B200"* ]]; then - platform="8xB200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B300"* ]]; then - platform="8xB300" -elif [[ "${gpus_on_node}" == *"4 NVIDIA GB300"* ]]; then - platform="4xGB300" -else - echo "Unsupported platform" - exit 0 -fi - -echo "Platform found: $platform" -echo "Running all_reduce_perf_nccl check on $(hostname)..." - -HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator -p $platform -n all_reduce_without_ib -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout --log-level info") -echo "Health checker output:" -echo "$HC_OUTPUT" -HC_STATUS=$(echo "$HC_OUTPUT" | awk '/^\s*{/,/^\s*}/' | jq -r '.status') - -echo "Health checker status: $HC_STATUS" -if [[ "$HC_STATUS" == "FAIL" ]]; then - echo "Health-checker reported status=FAIL." - exit 1 -elif [[ "$HC_STATUS" == "ERROR" ]]; then - echo "Health-checker reported status=ERROR." - exit 0 -else - echo "Health-checker passed or returned non-FAIL status." - exit 0 -fi diff --git a/helm/soperator-activechecks/scripts/cuda-samples.sh b/helm/soperator-activechecks/scripts/cuda-samples.sh deleted file mode 100644 index 507259ba3..000000000 --- a/helm/soperator-activechecks/scripts/cuda-samples.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#SBATCH --deadline="now+4hours" -#SBATCH --time=10:00 -#SBATCH --exclusive -#SBATCH --mem=0 - -platform="" -gpus_on_node=$(nvidia-smi --query-gpu=name --format=csv,noheader | sort | uniq -c) -if [[ "${gpus_on_node}" == *"8 NVIDIA H100"* ]]; then - platform="8xH100" -elif [[ "${gpus_on_node}" == *"8 NVIDIA H200"* ]]; then - platform="8xH200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B200"* ]]; then - platform="8xB200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B300"* ]]; then - platform="8xB300" -elif [[ "${gpus_on_node}" == *"4 NVIDIA GB300"* ]]; then - platform="4xGB300" -else - echo "Unsupported platform" - exit 0 -fi - -echo "Platform found: $platform" -echo "Running cuda samples check on $(hostname)..." -HC_OUTPUT_DIR="/opt/soperator-outputs/health_checker_cmd_stdout" -HC_OUTPUT=$(srun --container-image={{ include "activecheck.image.pyxis" . }} \ - --container-mounts=$(which health-checker):/usr/local/bin/health-checker,$HC_OUTPUT_DIR:$HC_OUTPUT_DIR \ - bash -l -c "health-checker run -e soperator -p $platform -n deviceQuery,vectorAdd,simpleMultiGPU,p2pBandwidthLatencyTest -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") - -echo "Health checker output:" -echo "$HC_OUTPUT" -HC_STATUS=$(echo "$HC_OUTPUT" | awk '/^\s*{/,/^\s*}/' | jq -r '.status') - -echo "Health checker status: $HC_STATUS" -if [[ "$HC_STATUS" == "FAIL" ]]; then - echo "Health-checker reported status=FAIL." - exit 1 -elif [[ "$HC_STATUS" == "ERROR" ]]; then - echo "Health-checker reported status=ERROR." - exit 0 -else - echo "Health-checker passed or returned non-FAIL status." - exit 0 -fi diff --git a/helm/soperator-activechecks/scripts/dcgmi-diag-r2.sh b/helm/soperator-activechecks/scripts/dcgmi-diag-r2.sh index 8197a352c..6127f410a 100644 --- a/helm/soperator-activechecks/scripts/dcgmi-diag-r2.sh +++ b/helm/soperator-activechecks/scripts/dcgmi-diag-r2.sh @@ -23,7 +23,7 @@ fi echo "Platform found: $platform" echo "Running dcgmi_diag_r2 check on $(hostname)..." -HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator -p $platform -n dcgmi_diag_r2 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") +HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -i dcgmi_diag_r2 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") echo "Health checker output:" echo "$HC_OUTPUT" diff --git a/helm/soperator-activechecks/scripts/dcgmi-diag-r3.sh b/helm/soperator-activechecks/scripts/dcgmi-diag-r3.sh index 39019578f..a53d9097d 100644 --- a/helm/soperator-activechecks/scripts/dcgmi-diag-r3.sh +++ b/helm/soperator-activechecks/scripts/dcgmi-diag-r3.sh @@ -23,7 +23,7 @@ fi echo "Platform found: $platform" echo "Running dcgmi_diag_r3 check on $(hostname)..." -HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -n dcgmi_diag_r3 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") +HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator-acceptance -p $platform -i dcgmi_diag_r3 -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") echo "Health checker output:" echo "$HC_OUTPUT" diff --git a/helm/soperator-activechecks/scripts/all-reduce-perf-nccl-with-ib.sh b/helm/soperator-activechecks/scripts/gpu-checks.sh similarity index 80% rename from helm/soperator-activechecks/scripts/all-reduce-perf-nccl-with-ib.sh rename to helm/soperator-activechecks/scripts/gpu-checks.sh index 008f80303..1dda2e36b 100644 --- a/helm/soperator-activechecks/scripts/all-reduce-perf-nccl-with-ib.sh +++ b/helm/soperator-activechecks/scripts/gpu-checks.sh @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --deadline="now+4hours" +#SBATCH --deadline="now+12hours" #SBATCH --time=30:00 #SBATCH --exclusive #SBATCH --mem=0 @@ -22,8 +22,8 @@ else fi echo "Platform found: $platform" -echo "Running all_reduce_perf_nccl check on $(hostname)..." -HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator -p $platform -n all_reduce_with_ib -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout --log-level info") +echo "Running GPU checks on $(hostname)..." +HC_OUTPUT=$(srun --cpu-bind=cores sudo bash -l -c "health-checker run -e soperator_active_checks -p $platform -x '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") echo "Health checker output:" echo "$HC_OUTPUT" diff --git a/helm/soperator-activechecks/scripts/gpu-fryer.sh b/helm/soperator-activechecks/scripts/gpu-fryer.sh deleted file mode 100644 index 7796e7c3a..000000000 --- a/helm/soperator-activechecks/scripts/gpu-fryer.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#SBATCH --deadline="now+8hours" -#SBATCH --time=20:00 -#SBATCH --exclusive -#SBATCH --mem=0 - -platform="" -gpus_on_node=$(nvidia-smi --query-gpu=name --format=csv,noheader | sort | uniq -c) -if [[ "${gpus_on_node}" == *"8 NVIDIA H100"* ]]; then - platform="8xH100" -elif [[ "${gpus_on_node}" == *"8 NVIDIA H200"* ]]; then - platform="8xH200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B200"* ]]; then - platform="8xB200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B300"* ]]; then - platform="8xB300" -elif [[ "${gpus_on_node}" == *"4 NVIDIA GB300"* ]]; then - platform="4xGB300" -else - echo "Unsupported platform" - exit 0 -fi - -echo "Platform found: $platform" -echo "Running gpu_fryer check on $(hostname)..." -HC_OUTPUT_DIR="/opt/soperator-outputs/health_checker_cmd_stdout" -HC_OUTPUT=$(srun --container-image={{ include "activecheck.image.pyxis" . }} \ - --container-mounts=$(which health-checker):/usr/local/bin/health-checker,$HC_OUTPUT_DIR:$HC_OUTPUT_DIR \ - bash -l -c "health-checker run -e soperator -p $platform -n gpu_fryer -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") - -echo "Health checker output:" -echo "$HC_OUTPUT" -HC_STATUS=$(echo "$HC_OUTPUT" | awk '/^\s*{/,/^\s*}/' | jq -r '.status') - -echo "Health checker status: $HC_STATUS" -if [[ "$HC_STATUS" == "FAIL" ]]; then - echo "Health-checker reported status=FAIL." - exit 1 -elif [[ "$HC_STATUS" == "ERROR" ]]; then - echo "Health-checker reported status=ERROR." - exit 0 -else - echo "Health-checker passed or returned non-FAIL status." - exit 0 -fi diff --git a/helm/soperator-activechecks/scripts/ib-gpu-perf.sh b/helm/soperator-activechecks/scripts/ib-gpu-perf.sh index e1509fce1..6daad8575 100644 --- a/helm/soperator-activechecks/scripts/ib-gpu-perf.sh +++ b/helm/soperator-activechecks/scripts/ib-gpu-perf.sh @@ -26,7 +26,7 @@ echo "Running ib_gpu_perf check on $(hostname)..." HC_OUTPUT_DIR="/opt/soperator-outputs/health_checker_cmd_stdout" HC_OUTPUT=$(srun --container-image={{ include "activecheck.image.pyxis" . }} \ --container-mounts=$(which health-checker):/usr/local/bin/health-checker,$HC_OUTPUT_DIR:$HC_OUTPUT_DIR --cpu-bind=cores \ - bash -l -c "health-checker run -e soperator -p $platform -n '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") + bash -l -c "health-checker run -e soperator_active_checks -p $platform -i '^ib_write_bw_gpu.*$,^ib_send_lat_gpu.*$,^ib_read_lat_gpu.*$' -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") echo "Health checker output:" echo "$HC_OUTPUT" diff --git a/helm/soperator-activechecks/scripts/mem-perf.sh b/helm/soperator-activechecks/scripts/mem-perf.sh deleted file mode 100644 index df67dc928..000000000 --- a/helm/soperator-activechecks/scripts/mem-perf.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#SBATCH --deadline="now+8hours" -#SBATCH --time=15:00 -#SBATCH --exclusive -#SBATCH --mem=0 - -platform="" -gpus_on_node=$(nvidia-smi --query-gpu=name --format=csv,noheader | sort | uniq -c) -if [[ "${gpus_on_node}" == *"8 NVIDIA H100"* ]]; then - platform="8xH100" -elif [[ "${gpus_on_node}" == *"8 NVIDIA H200"* ]]; then - platform="8xH200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B200"* ]]; then - platform="8xB200" -elif [[ "${gpus_on_node}" == *"8 NVIDIA B300"* ]]; then - platform="8xB300" -elif [[ "${gpus_on_node}" == *"4 NVIDIA GB300"* ]]; then - platform="4xGB300" -else - echo "Unsupported platform" - exit 0 -fi - -echo "Platform found: $platform" -echo "Running mem_perf check on $(hostname)..." -HC_OUTPUT_DIR="/opt/soperator-outputs/health_checker_cmd_stdout" -HC_OUTPUT=$(srun --container-image={{ include "activecheck.image.pyxis" . }} \ - --container-mounts=$(which health-checker):/usr/local/bin/health-checker,$HC_OUTPUT_DIR:$HC_OUTPUT_DIR --cpu-bind=cores \ - bash -l -c "health-checker run -e soperator -p $platform -n mem_bw,mem_lat -f json-partial --tests-stdout-path /opt/soperator-outputs/health_checker_cmd_stdout") - -echo "Health checker output:" -echo "$HC_OUTPUT" -HC_STATUS=$(echo "$HC_OUTPUT" | awk '/^\s*{/,/^\s*}/' | jq -r '.status') - -echo "Health checker status: $HC_STATUS" -if [[ "$HC_STATUS" == "FAIL" ]]; then - echo "Health-checker reported status=FAIL." - exit 1 -elif [[ "$HC_STATUS" == "ERROR" ]]; then - echo "Health-checker reported status=ERROR." - exit 0 -else - echo "Health-checker passed or returned non-FAIL status." - exit 0 -fi diff --git a/helm/soperator-activechecks/tests/reactions_conflict_test.yaml b/helm/soperator-activechecks/tests/reactions_conflict_test.yaml index 7b3f9a86d..3fbf461bd 100644 --- a/helm/soperator-activechecks/tests/reactions_conflict_test.yaml +++ b/helm/soperator-activechecks/tests/reactions_conflict_test.yaml @@ -41,11 +41,11 @@ tests: asserts: - notFailedTemplate: {} - - it: gpu-fryer should fail if both commentSlurmNode and drainSlurmNode are set under failureReactions + - it: gpu-checks should fail if both commentSlurmNode and drainSlurmNode are set under failureReactions template: templates/active-checks.yaml set: checks: - gpu-fryer: + gpu-checks: failureReactions: commentSlurmNode: commentPrefix: "[node_problem]" @@ -53,13 +53,13 @@ tests: drainReasonPrefix: "[node_problem]" asserts: - failedTemplate: - errorMessage: "checks.gpu-fryer.failureReactions: cannot set both commentSlurmNode and drainSlurmNode simultaneously" + errorMessage: "checks.gpu-checks.failureReactions: cannot set both commentSlurmNode and drainSlurmNode simultaneously" - - it: gpu-fryer should render when only commentSlurmNode is set + - it: gpu-checks should render when only commentSlurmNode is set template: templates/active-checks.yaml set: checks: - gpu-fryer: + gpu-checks: failureReactions: commentSlurmNode: commentPrefix: "[node_problem]" @@ -67,11 +67,11 @@ tests: asserts: - notFailedTemplate: {} - - it: gpu-fryer should render when only drainSlurmNode is set + - it: gpu-checks should render when only drainSlurmNode is set template: templates/active-checks.yaml set: checks: - gpu-fryer: + gpu-checks: failureReactions: commentSlurmNode: null drainSlurmNode: @@ -79,11 +79,11 @@ tests: asserts: - notFailedTemplate: {} - - it: gpu-fryer should render when both reaction nodes are explicitly null + - it: gpu-checks should render when both reaction nodes are explicitly null template: templates/active-checks.yaml set: checks: - gpu-fryer: + gpu-checks: failureReactions: commentSlurmNode: null drainSlurmNode: null diff --git a/helm/soperator-activechecks/tests/slurmjob_gpu_test.yaml b/helm/soperator-activechecks/tests/slurmjob_gpu_test.yaml index 7cb7b8f4b..c07f7baa8 100644 --- a/helm/soperator-activechecks/tests/slurmjob_gpu_test.yaml +++ b/helm/soperator-activechecks/tests/slurmjob_gpu_test.yaml @@ -5,7 +5,7 @@ tests: - it: should render default SBATCH_GPUS_PER_NODE for GPU slurm jobs documentSelector: path: metadata.name - value: soperator-cuda-samples + value: soperator-gpu-checks asserts: - contains: path: spec.slurmJobSpec.jobContainer.env @@ -16,7 +16,7 @@ tests: - it: should render overridden SBATCH_GPUS_PER_NODE for GPU slurm jobs documentSelector: path: metadata.name - value: soperator-cuda-samples + value: soperator-gpu-checks set: slurmJob: gpusPerNode: "4" diff --git a/helm/soperator-activechecks/tests/slurmjob_spec_test.yaml b/helm/soperator-activechecks/tests/slurmjob_spec_test.yaml index 98172637a..64a227ea7 100644 --- a/helm/soperator-activechecks/tests/slurmjob_spec_test.yaml +++ b/helm/soperator-activechecks/tests/slurmjob_spec_test.yaml @@ -3,10 +3,10 @@ templates: - templates/active-checks.yaml tests: - - it: should render cuda-samples active check + - it: should render gpu-checks active check documentSelector: path: metadata.name - value: soperator-cuda-samples + value: soperator-gpu-checks asserts: - isKind: of: ActiveCheck @@ -21,10 +21,10 @@ tests: value: false - contains: path: spec.dependsOn - content: soperator-prepull-container-image + content: soperator-manage-jail-state - matchRegex: path: spec.slurmJobSpec.sbatchScript pattern: 'health-checker run' - matchRegex: path: spec.slurmJobSpec.sbatchScript - pattern: 'Running cuda samples check' + pattern: 'Running GPU checks' diff --git a/helm/soperator-activechecks/values.yaml b/helm/soperator-activechecks/values.yaml index 72cb00226..fd0049910 100644 --- a/helm/soperator-activechecks/values.yaml +++ b/helm/soperator-activechecks/values.yaml @@ -54,60 +54,17 @@ slurmJob: # Slurm consumes this as SBATCH_GPUS_PER_NODE for GPU ActiveChecks. gpusPerNode: "8" checks: - all-reduce-perf-nccl-in-docker: - enabled: true - checkType: "slurmJob" - dependsOn: - - "manage-jail-state" - - "wait-for-soperatorchecks-srun-ready" - - "wait-for-topology" - - "prepull-container-image" - suspend: true - runAfterCreation: true - slurmJobSpec: - sbatchScriptFile: "scripts/all-reduce-perf-nccl-in-docker.sh" - eachWorkerJobs: true - maxNumberOfJobs: 200 - jobContainer: - extraEnv: - - name: "SBATCH_GPUS_PER_NODE" - value: "{{ .Values.slurmJob.gpusPerNode }}" - failureReactions: - drainSlurmNode: - drainReasonPrefix: "[node_problem]" - all-reduce-perf-nccl-with-ib: - enabled: true - checkType: "slurmJob" - dependsOn: - - "manage-jail-state" - - "wait-for-soperatorchecks-srun-ready" - - "wait-for-topology" - schedule: "45 0-18/6 * * *" - suspend: false - runAfterCreation: true - slurmJobSpec: - sbatchScriptFile: "scripts/all-reduce-perf-nccl-with-ib.sh" - eachWorkerJobs: true - maxNumberOfJobs: 200 - jobContainer: - extraEnv: - - name: "SBATCH_GPUS_PER_NODE" - value: "{{ .Values.slurmJob.gpusPerNode }}" - failureReactions: - drainSlurmNode: - drainReasonPrefix: "[node_problem]" - all-reduce-perf-nccl-without-ib: + gpu-checks: enabled: true checkType: "slurmJob" dependsOn: - "manage-jail-state" - "wait-for-soperatorchecks-srun-ready" - "wait-for-topology" - schedule: "25 5-23/6 * * *" suspend: false runAfterCreation: true slurmJobSpec: - sbatchScriptFile: "scripts/all-reduce-perf-nccl-without-ib.sh" + sbatchScriptFile: "scripts/gpu-checks.sh" eachWorkerJobs: true jobContainer: extraEnv: @@ -116,7 +73,7 @@ checks: failureReactions: drainSlurmNode: drainReasonPrefix: "[node_problem]" - cuda-samples: + all-reduce-perf-nccl-in-docker: enabled: true checkType: "slurmJob" dependsOn: @@ -124,12 +81,12 @@ checks: - "wait-for-soperatorchecks-srun-ready" - "wait-for-topology" - "prepull-container-image" - schedule: "30 0-18/6 * * *" - suspend: false + suspend: true runAfterCreation: true slurmJobSpec: - sbatchScriptFile: "scripts/cuda-samples.sh" + sbatchScriptFile: "scripts/all-reduce-perf-nccl-in-docker.sh" eachWorkerJobs: true + maxNumberOfJobs: 200 jobContainer: extraEnv: - name: "SBATCH_GPUS_PER_NODE" @@ -138,7 +95,7 @@ checks: drainSlurmNode: drainReasonPrefix: "[node_problem]" dcgmi-diag-r2: - enabled: true + enabled: false checkType: "slurmJob" dependsOn: - "manage-jail-state" @@ -158,7 +115,7 @@ checks: drainSlurmNode: drainReasonPrefix: "[node_problem]" dcgmi-diag-r3: - enabled: true + enabled: false checkType: "slurmJob" dependsOn: - "manage-jail-state" @@ -180,41 +137,15 @@ checks: enabled: true checkType: "slurmJob" dependsOn: - - "manage-jail-state" - "all-reduce-perf-nccl-in-docker" - - "all-reduce-perf-nccl-with-ib" - - "all-reduce-perf-nccl-without-ib" - - "cuda-samples" - - "dcgmi-diag-r2" - - "dcgmi-diag-r3" - - "gpu-fryer" - - "ib-gpu-perf" - - "mem-perf" - suspend: true - runAfterCreation: true - slurmJobSpec: - sbatchScriptFile: "scripts/ensure-healthy-nodes.sh" - gpu-fryer: - enabled: true - checkType: "slurmJob" - dependsOn: + - "gpu-checks" - "manage-jail-state" - "wait-for-soperatorchecks-srun-ready" - "wait-for-topology" - - "prepull-container-image" - schedule: "15 9-21/12 * * *" - suspend: false + suspend: true runAfterCreation: true slurmJobSpec: - sbatchScriptFile: "scripts/gpu-fryer.sh" - eachWorkerJobs: true - jobContainer: - extraEnv: - - name: "SBATCH_GPUS_PER_NODE" - value: "{{ .Values.slurmJob.gpusPerNode }}" - failureReactions: - drainSlurmNode: - drainReasonPrefix: "[node_problem]" + sbatchScriptFile: "scripts/ensure-healthy-nodes.sh" ib-gpu-perf: enabled: true checkType: "slurmJob" @@ -237,27 +168,6 @@ checks: failureReactions: commentSlurmNode: commentPrefix: "[node_problem]" - mem-perf: - enabled: true - checkType: "slurmJob" - dependsOn: - - "manage-jail-state" - - "wait-for-soperatorchecks-srun-ready" - - "wait-for-topology" - - "prepull-container-image" - schedule: "20 10-22/12 * * *" - suspend: false - runAfterCreation: true - slurmJobSpec: - sbatchScriptFile: "scripts/mem-perf.sh" - eachWorkerJobs: true - jobContainer: - extraEnv: - - name: "SBATCH_GPUS_PER_NODE" - value: "{{ .Values.slurmJob.gpusPerNode }}" - failureReactions: - drainSlurmNode: - drainReasonPrefix: "[node_problem]" ssh-check: enabled: true checkType: "k8sJob" @@ -423,35 +333,6 @@ checks: value: "ansible.posix.jsonl" - name: "ANSIBLE_FORCE_COLOR" value: "0" - upgrade-health-checker: - enabled: true - checkType: "k8sJob" - dependsOn: - - "create-user-soperatorchecks" - suspend: true - runAfterCreation: false - k8sJobSpec: - jobContainer: - appArmorProfile: unconfined - workingDir: /opt/ansible - image: "{{ .Values.images.sansible }}" - command: ["/opt/bin/sansible_entrypoint.sh"] - args: - - ansible-playbook - - -i - - inventory/ - - -c - - local - - -e - - ansible_connection=community.general.chroot - - -e - - ansible_host=/mnt/jail - - -e - - ansible_python_interpreter=/usr/bin/python3 - - -e - - "nc_health_checker_version=1.1.15-b3af5269" - - nc-health-checker.yml - - --diff wait-for-topology: enabled: true checkType: "k8sJob"