[DRAFT] poc seamless worker update

Makefile

@@ -59,7 +59,7 @@ NFS_IMAGE_TAG		= $(shell $(YQ) '.image.tag' helm/nfs-server/values.yaml)
 OPERATOR_IMAGE_TAG  = $(VERSION)
 
 ifeq ($(shell uname), Darwin)
-    SED_COMMAND = sed -i '' -e
+    SED_COMMAND = sed -i -e
 else
     SED_COMMAND = sed -i -e
 endif

api/v1alpha1/nodeset_types.go

@@ -7,6 +7,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"nebius.ai/slurm-operator/internal/consts"
 )
 
 // +kubebuilder:object:root=true
@@ -300,6 +302,12 @@ type NodeSetSpec struct {
 	CustomInitContainers []corev1.Container `json:"customInitContainers,omitempty"`
 
 	// endregion Scheduling
+
+	// UpdateStrategy for the advanced StatefulSet
+	//
+	// +kubebuilder:validation:Enum=rollingUpdate;onDelete
+	// +kubebuilder:default="rollingUpdate"
+	UpdateStrategy consts.UpdateStrategy `json:"updateStrategy"`
 }
 
 // ContainerSlurmdSpec defines the Slurm worker daemon configuration

cmd/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"os"
@@ -52,9 +53,12 @@ import (
 	"nebius.ai/slurm-operator/internal/controller/clustercontroller"
 	"nebius.ai/slurm-operator/internal/controller/nodeconfigurator"
 	"nebius.ai/slurm-operator/internal/controller/nodesetcontroller"
+	"nebius.ai/slurm-operator/internal/controller/soperatorchecks"
 	"nebius.ai/slurm-operator/internal/controller/topologyconfcontroller"
+	"nebius.ai/slurm-operator/internal/controller/updatecontroller"
 	"nebius.ai/slurm-operator/internal/controllersenabled"
 	metricsopts "nebius.ai/slurm-operator/internal/metrics"
+	"nebius.ai/slurm-operator/internal/slurmapi"
 	webhookv1 "nebius.ai/slurm-operator/internal/webhook/v1"
 	webhookv1alpha1 "nebius.ai/slurm-operator/internal/webhook/v1alpha1"
 	//+kubebuilder:scaffold:imports
@@ -180,7 +184,7 @@ func main() {
 		controllersSpec = controllersFlag
 		controllersSource = "flag"
 	}
-	availableControllers := []string{"cluster", "nodeconfigurator", "nodeset", "topology"}
+	availableControllers := []string{"cluster", "nodeconfigurator", "nodeset", "rollingupdate", "topology"}
 	controllersSet, err := controllersenabled.New(
 		controllersSpec,
 		availableControllers,
@@ -307,6 +311,29 @@ func main() {
 	}
 	// endregion Reconciler/NodeSet
 
+	slurmAPIClients := slurmapi.NewClientSet(context.Background())
+
+	if controllersSet.Enabled("rollingupdate") {
+		if err = soperatorchecks.NewSlurmAPIClientsController(
+			mgr.GetClient(),
+			mgr.GetScheme(),
+			mgr.GetEventRecorderFor(soperatorchecks.SlurmAPIClientsControllerName),
+			slurmAPIClients,
+		).SetupWithManager(mgr, maxConcurrency, cacheSyncTimeout); err != nil {
+			cli.Fail(setupLog, err, "unable to create slurm api clients controller", "controller", soperatorchecks.SlurmAPIClientsControllerName)
+		}
+
+		if err = updatecontroller.NewRollingUpdateReconciler(
+			mgr.GetClient(),
+			mgr.GetScheme(),
+			mgr.GetEventRecorderFor(updatecontroller.RollingUpdateControllerName),
+			slurmAPIClients,
+		).
+			SetupWithManager(mgr, maxConcurrency, cacheSyncTimeout); err != nil {
+			cli.Fail(setupLog, err, "unable to create controller", "controller", updatecontroller.RollingUpdateControllerName)
+		}
+	}
+
 	// region Reconciler/Topology
 	if controllersSet.Enabled("topology") {
 		if err = topologyconfcontroller.NewNodeTopologyReconciler(

cmd/soperatorchecks/main.go

@@ -282,7 +282,6 @@ func main() {
 			mgr.GetScheme(),
 			mgr.GetEventRecorderFor(soperatorchecks.SlurmAPIClientsControllerName),
 			slurmAPIClients,
-			corev1.NodeConditionType(maintenanceConditionType),
 		).SetupWithManager(mgr, maxConcurrency, cacheSyncTimeout); err != nil {
 			cli.Fail(setupLog, err, "unable to create slurm api clients controller", "controller", soperatorchecks.SlurmAPIClientsControllerName)
 		}

config/crd/bases/slurm.nebius.ai_nodesets.yaml

@@ -12083,6 +12083,13 @@ spec:
                       Defaults to "root", which preserves the single-fabric behavior.
                     type: string
                 type: object
+              updateStrategy:
+                default: rollingUpdate
+                description: UpdateStrategy for the advanced StatefulSet
+                enum:
+                - rollingUpdate
+                - onDelete
+                type: string
               workerAnnotations:
                 additionalProperties:
                   type: string
@@ -12105,6 +12112,7 @@ spec:
             required:
             - munge
             - slurmd
+            - updateStrategy
             type: object
           status:
             description: NodeSetStatus defines the observed state of SlurmCluster

helm/nodesets/templates/nodeset.yaml

@@ -31,6 +31,8 @@ spec:
   maxUnavailable: {{ . }}
   {{- end }}
 
+  updateStrategy: {{ (.updateStrategy | default "rollingUpdate") | quote }}
+
   {{- with .maxConcurrentStartup }}
   maxConcurrentStartup: {{ . }}
   {{- end }}
@@ -263,7 +265,7 @@ spec:
   sssdLdapCAConfigMapRefName: {{ . | quote }}
   {{- end }}
 
-  {{- with (.configMapRefSupervisord | default "") }}
+  {{- with (.configMapRefSupervisord | default $.Values.configMapRefSupervisord | default "") }}
   configMapRefSupervisord: {{ . | quote }}
   {{- end }}
 

helm/nodesets/tests/basic_test.yaml

@@ -97,6 +97,63 @@ tests:
           path: kind
           value: NodeSet
 
+  - it: should use chart-level supervisord ConfigMap ref when NodeSet does not override it
+    set:
+      configMapRefSupervisord: custom-supervisord-config-4.0.2
+      nodesets:
+        - name: test-workers
+          slurmd:
+            image:
+              repository: "test/slurm"
+            resources:
+              cpu: "1"
+              memory: "1Gi"
+            volumes:
+              spool:
+                emptyDir: {}
+              jail:
+                emptyDir: {}
+              jailSubMounts: []
+          munge:
+            image:
+              repository: "test/munge"
+            resources:
+              cpu: "100m"
+              memory: "128Mi"
+    asserts:
+      - equal:
+          path: spec.configMapRefSupervisord
+          value: custom-supervisord-config-4.0.2
+
+  - it: should prefer NodeSet supervisord ConfigMap ref over chart-level default
+    set:
+      configMapRefSupervisord: custom-supervisord-config-4.0.2
+      nodesets:
+        - name: test-workers
+          configMapRefSupervisord: per-nodeset-supervisord-config
+          slurmd:
+            image:
+              repository: "test/slurm"
+            resources:
+              cpu: "1"
+              memory: "1Gi"
+            volumes:
+              spool:
+                emptyDir: {}
+              jail:
+                emptyDir: {}
+              jailSubMounts: []
+          munge:
+            image:
+              repository: "test/munge"
+            resources:
+              cpu: "100m"
+              memory: "128Mi"
+    asserts:
+      - equal:
+          path: spec.configMapRefSupervisord
+          value: per-nodeset-supervisord-config
+
   - it: should include common labels
     set:
       nodesets:

helm/nodesets/tests/custom_values_test.yaml

@@ -8,6 +8,7 @@ tests:
         - name: custom-workers
           replicas: 10
           priorityClass: "custom-priority"
+          updateStrategy: onDelete
           slurmd:
             image:
               repository: "custom/slurm"
@@ -48,6 +49,9 @@ tests:
       - equal:
           path: spec.priorityClass
           value: "custom-priority"
+      - equal:
+          path: spec.updateStrategy
+          value: "onDelete"
       - equal:
           path: spec.slurmd.image.repository
           value: "custom/slurm"
@@ -99,6 +103,9 @@ tests:
       - equal:
           path: spec.replicas
           value: 1  # default value
+      - equal:
+          path: spec.updateStrategy
+          value: "rollingUpdate"
       - notExists:
           path: spec.priorityClass
       - notExists:

helm/nodesets/values.yaml

@@ -6,6 +6,9 @@
 nameOverride: ""
 fullnameOverride: ""
 clusterName: ""
+# Default custom SupervisorD config ConfigMap for NodeSets.
+# Individual NodeSets can override this with nodesets[*].configMapRefSupervisord.
+configMapRefSupervisord: ""
 # Priority Classes configuration
 # Define priority classes that can be used by NodeSets
 priorityClasses:
@@ -50,6 +53,10 @@ nodesets:
     # Could be a count (number) or percent (string)
     # Optional, defaults to 20%
     maxUnavailable: 1
+    # Update strategy for the worker StatefulSet.
+    # Valid values: rollingUpdate, onDelete.
+    # Optional, defaults to rollingUpdate
+    updateStrategy: rollingUpdate
     # Maximum number of worker pods that can be created in parallel during
     # initial scale-out, to avoid overloading the Slurm controller with
     # simultaneous slurmd registrations.

helm/soperator-crds/templates/slurmcluster-crd.yaml

@@ -27177,6 +27177,13 @@ spec:
                       Defaults to "root", which preserves the single-fabric behavior.
                     type: string
                 type: object
+              updateStrategy:
+                default: rollingUpdate
+                description: UpdateStrategy for the advanced StatefulSet
+                enum:
+                - rollingUpdate
+                - onDelete
+                type: string
               workerAnnotations:
                 additionalProperties:
                   type: string
@@ -27199,6 +27206,7 @@ spec:
             required:
             - munge
             - slurmd
+            - updateStrategy
             type: object
           status:
             description: NodeSetStatus defines the observed state of SlurmCluster

helm/soperator-custom-configmaps/templates/_helpers.tpl

@@ -12,6 +12,15 @@ Create chart name and version as used by the chart label.
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Create a ConfigMap name with a chart-version suffix.
+*/}}
+{{- define "soperator-custom-configmaps.versionedName" -}}
+{{- $name := index . 0 -}}
+{{- $version := index . 1 -}}
+{{- printf "%s-%s" $name $version | lower | replace "+" "-" | replace "_" "-" | trunc 253 | trimSuffix "-" }}
+{{- end }}
+
 {{/*
 Common labels
 */}}
@@ -24,6 +33,15 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
+{{/*
+Stable labels for legacy kept resources. These intentionally exclude chart/app
+version labels so chart upgrades do not keep patching the old ConfigMap.
+*/}}
+{{- define "soperator-custom-configmaps.stableLabels" -}}
+{{ include "soperator-custom-configmaps.selectorLabels" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
 {{/*
 Selector labels
 */}}

helm/soperator-custom-configmaps/templates/configmap-supervisord.yaml

@@ -1,14 +1,60 @@
 {{- if .Values.configMaps.supervisord.enabled }}
+{{- $baseName := .Values.configMaps.supervisord.name }}
+{{- $versionedName := include "soperator-custom-configmaps.versionedName" (list $baseName .Chart.Version) }}
+{{- $keepLegacy := true }}
+{{- if hasKey .Values.configMaps.supervisord "keepLegacy" }}
+{{- $keepLegacy = .Values.configMaps.supervisord.keepLegacy }}
+{{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Values.configMaps.supervisord.name }}
+  name: {{ $versionedName }}
   namespace: {{ .Values.namespace }}
+  annotations:
+    helm.sh/resource-policy: keep
+    slurm.nebius.ai/dependency-version: name
   labels:
     {{- include "soperator-custom-configmaps.labels" . | nindent 4 }}
+data:
+  {{- range .Values.configMaps.supervisord.files }}
+  {{ . }}: |
+{{ tpl ($.Files.Get (printf "config-files/%s" .)) $ | indent 4 }}
+{{- end }}
+{{- if and $keepLegacy (ne $baseName $versionedName) }}
+{{- $legacyConfigMap := lookup "v1" "ConfigMap" .Values.namespace $baseName }}
+{{- $legacyLabels := include "soperator-custom-configmaps.stableLabels" . | fromYaml }}
+{{- $legacyAnnotations := dict "helm.sh/resource-policy" "keep" }}
+{{- if $legacyConfigMap }}
+{{- with $legacyConfigMap.metadata.labels }}
+{{- $legacyLabels = deepCopy . }}
+{{- end }}
+{{- with $legacyConfigMap.metadata.annotations }}
+{{- $legacyAnnotations = mergeOverwrite (deepCopy .) $legacyAnnotations }}
+{{- end }}
+{{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $baseName }}
+  namespace: {{ .Values.namespace }}
+  annotations:
+    {{- toYaml $legacyAnnotations | nindent 4 }}
+  labels:
+    {{- toYaml $legacyLabels | nindent 4 }}
+{{- if and $legacyConfigMap $legacyConfigMap.data }}
+data:
+  {{- toYaml $legacyConfigMap.data | nindent 2 }}
+{{- else }}
 data:
   {{- range .Values.configMaps.supervisord.files }}
   {{ . }}: |
 {{ tpl ($.Files.Get (printf "config-files/%s" .)) $ | indent 4 }}
   {{- end }}
 {{- end }}
+{{- if and $legacyConfigMap $legacyConfigMap.binaryData }}
+binaryData:
+  {{- toYaml $legacyConfigMap.binaryData | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}