Add support for enabling Consul dataplane as a sidecar container (#4678)

* Add support for enabling Consul dataplane as a sidecar container

* Add sidecar initial probe check delay configuration

* Add sidecar probe period, failure threshold, and timeout configurations

* Update default sidecar probe configurations for improved reliability

* Add tests for Consul dataplane sidecar lifecycle configuration and probe handling

* added changelog

* Add configuration for Consul dataplane as sidecar init container in values.yaml

* Bump kind version to v0.27.0 in CI inputs for compatibility updates

Author: nitin-sachdev-29

.changelog/4678.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+Added boolean annotation "consul.hashicorp.com/enable-consul-dataplane-as-sidecar" for registering consul-dataplane as init container so that consul-dataplane container is initialised and started before application container. Default value is "false" i.e the feature is disabled by default. Also made the probe properties configurable through annotations.
+```

acceptance/ci-inputs/kind-inputs.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: MPL-2.0
 
-kindVersion: v0.23.0
+kindVersion: v0.27.0
 kindNodeImage: kindest/node:v1.30.2@sha256:ecfe5841b9bee4fe9690f49c118c33629fa345e3350a0c67a5a34482a99d6bba
 kubectlVersion: v1.30.2

charts/consul/templates/connect-inject-deployment.yaml

@@ -284,6 +284,17 @@ spec:
                 -enable-auto-encrypt \
                 {{- end }}
                 -enable-telemetry-collector={{ .Values.global.metrics.enableTelemetryCollector}}  \
+
+                {{- if .Values.connectInject.dataplaneAsSidecarInitContainer.defaultEnabled }}
+                -default-enable-consul-dataplane-as-sidecar=true \
+                -default-sidecar-probe-check-initial-delay-seconds={{ .Values.connectInject.dataplaneAsSidecarInitContainer.initialProbeCheckDelaySeconds }} \
+                -default-sidecar-probe-period-seconds={{ .Values.connectInject.dataplaneAsSidecarInitContainer.probeCheckPeriodSeconds }} \
+                -default-sidecar-probe-failure-threshold="{{ .Values.connectInject.dataplaneAsSidecarInitContainer.probeCheckFailureThreshold }}" \
+                -default-sidecar-probe-check-timeout-seconds="{{ .Values.connectInject.dataplaneAsSidecarInitContainer.probeCheckTimeoutSeconds }}" \
+                {{- else }}
+                -default-enable-consul-dataplane-as-sidecar=false \
+                {{- end }}
+
           startupProbe:
             httpGet:
               path: /readyz/ready

charts/consul/values.yaml

@@ -3015,6 +3015,31 @@ connectInject:
     # A value of zero disables the probe.
     defaultLivenessFailureSeconds: 0
 
+  dataplaneAsSidecarInitContainer:
+    # If true, the consul-dataplane will run as an init container with the restart policy set to "Always" and using startup probe to determine readiness.
+    # Also, Kubelet will wait for the consul-dataplane init container to be ready before starting the application container.
+    # @type: boolean
+    defaultEnabled: false
+
+    # Configures how long the k8s startup probe will wait initially before performing the first check.
+    # @type: integer
+    initialProbeCheckDelaySeconds: 1
+
+    # Configures how often the k8s startup probe will check the readiness of the consul-dataplane init container.
+    # @type: integer
+    probeCheckPeriodSeconds: 1
+
+    # Configures the failure threshold for the k8s startup probe.
+    # @type: integer
+    probeCheckFailureThreshold: 10
+
+    # Configures the timeout for each k8s startup probe check.
+    # @type: integer
+    probeCheckTimeoutSeconds: 5
+
+
+
+
   # The resource settings for the Connect injected init container. If null, the resources
   # won't be set for the initContainer. The defaults are optimized for developer instances of
   # Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times.

control-plane/connect-inject/constants/annotations_and_labels.go

@@ -226,6 +226,12 @@ const (
 	// ManagedByValue is the value for keyManagedBy.
 	//TODO(zalimeni) rename this to ManagedByLegacyEndpointsValue.
 	ManagedByValue = "consul-k8s-endpoints-controller"
+
+	AnnotationEnableConsulDataplaneAsSidecar       = "consul.hashicorp.com/enable-consul-dataplane-as-sidecar"
+	AnnotationSidecarInitialProbeCheckDelaySeconds = "consul.hashicorp.com/sidecar-initial-probe-check-delay-seconds"
+	AnnotationSidecarProbePeriodSeconds            = "consul.hashicorp.com/sidecar-probe-period-seconds"
+	AnnotationSidecarProbeFailureThreshold         = "consul.hashicorp.com/sidecar-probe-failure-threshold"
+	AnnotationSidecarProbeCheckTimeoutSeconds      = "consul.hashicorp.com/sidecar-probe-check-timeout-seconds"
 )
 
 // Annotations used by Prometheus.

control-plane/connect-inject/lifecycle/lifecycle_configuration.go

@@ -14,13 +14,14 @@ import (
 
 // Config represents configuration common to connect-inject components related to proxy lifecycle management.
 type Config struct {
-	DefaultEnableProxyLifecycle         bool
-	DefaultEnableShutdownDrainListeners bool
-	DefaultShutdownGracePeriodSeconds   int
-	DefaultStartupGracePeriodSeconds    int
-	DefaultGracefulPort                 string
-	DefaultGracefulShutdownPath         string
-	DefaultGracefulStartupPath          string
+	DefaultEnableProxyLifecycle           bool
+	DefaultEnableShutdownDrainListeners   bool
+	DefaultShutdownGracePeriodSeconds     int
+	DefaultStartupGracePeriodSeconds      int
+	DefaultGracefulPort                   string
+	DefaultGracefulShutdownPath           string
+	DefaultGracefulStartupPath            string
+	DefaultEnableConsulDataplaneAsSidecar bool
 }
 
 // EnableProxyLifecycle returns whether proxy lifecycle management is enabled either via the default value in the meshWebhook, or if it's been
@@ -123,3 +124,17 @@ func (lc Config) GracefulStartupPath(pod corev1.Pod) string {
 
 	return lc.DefaultGracefulStartupPath
 }
+
+// EnableConsulDataplaneAsSidecar returns whether register consul-dataplane as sidecar in kubernetes is enabled either via the default value in the meshWebhook, or if it's been
+// overridden via the annotation.
+func (lc Config) EnableConsulDataplaneAsSidecar(pod corev1.Pod) (bool, error) {
+	enabled := lc.DefaultEnableConsulDataplaneAsSidecar
+	if raw, ok := pod.Annotations[constants.AnnotationEnableConsulDataplaneAsSidecar]; ok && raw != "" {
+		enableConsulDataplaneAsSidecar, err := strconv.ParseBool(raw)
+		if err != nil {
+			return false, fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableConsulDataplaneAsSidecar, raw, err)
+		}
+		enabled = enableConsulDataplaneAsSidecar
+	}
+	return enabled, nil
+}

control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go

@@ -468,3 +468,77 @@ func minimal() *corev1.Pod {
 		},
 	}
 }
+
+func TestLifecycleConfig_EnableConsulDataplaneAsSidecar(t *testing.T) {
+	cases := []struct {
+		Name            string
+		Pod             func(*corev1.Pod) *corev1.Pod
+		LifecycleConfig Config
+		Expected        bool
+		Err             string
+	}{
+		{
+			Name: "Enabled via meshWebhook default",
+			Pod: func(pod *corev1.Pod) *corev1.Pod {
+				return pod
+			},
+			LifecycleConfig: Config{
+				DefaultEnableConsulDataplaneAsSidecar: true,
+			},
+			Expected: true,
+			Err:      "",
+		},
+		{
+			Name: "Enabled via annotation",
+			Pod: func(pod *corev1.Pod) *corev1.Pod {
+				pod.Annotations[constants.AnnotationEnableConsulDataplaneAsSidecar] = "true"
+				return pod
+			},
+			LifecycleConfig: Config{
+				DefaultEnableConsulDataplaneAsSidecar: false,
+			},
+			Expected: true,
+			Err:      "",
+		},
+		{
+			Name: "Disabled via annotation",
+			Pod: func(pod *corev1.Pod) *corev1.Pod {
+				pod.Annotations[constants.AnnotationEnableConsulDataplaneAsSidecar] = "false"
+				return pod
+			},
+			LifecycleConfig: Config{
+				DefaultEnableConsulDataplaneAsSidecar: true,
+			},
+			Expected: false,
+			Err:      "",
+		},
+		{
+			Name: "Invalid annotation value",
+			Pod: func(pod *corev1.Pod) *corev1.Pod {
+				pod.Annotations[constants.AnnotationEnableConsulDataplaneAsSidecar] = "not-a-bool"
+				return pod
+			},
+			LifecycleConfig: Config{
+				DefaultEnableConsulDataplaneAsSidecar: false,
+			},
+			Expected: false,
+			Err:      "consul.hashicorp.com/enable-consul-dataplane-as-sidecar annotation value of not-a-bool was invalid: strconv.ParseBool: parsing \"not-a-bool\": invalid syntax",
+		},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.Name, func(t *testing.T) {
+			require := require.New(t)
+			lc := tt.LifecycleConfig
+
+			actual, err := lc.EnableConsulDataplaneAsSidecar(*tt.Pod(minimal()))
+
+			if tt.Err == "" {
+				require.Equal(tt.Expected, actual)
+				require.NoError(err)
+			} else {
+				require.EqualError(err, tt.Err)
+			}
+		})
+	}
+}

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -276,6 +276,32 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		},
 		ReadOnlyRootFilesystem: ptr.To(true),
 	}
+	enableConsulDataplaneAsSidecar, err := w.LifecycleConfig.EnableConsulDataplaneAsSidecar(pod)
+	if err != nil {
+		return corev1.Container{}, err
+	}
+	if enableConsulDataplaneAsSidecar {
+		restartPolicy := corev1.ContainerRestartPolicyAlways
+		container.RestartPolicy = &restartPolicy
+
+		// Configure the startup probe to check the sidecar proxy health.
+		container.StartupProbe = &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{
+						// Absolute path to the binary from the Dockerfile
+						"/usr/local/bin/consul-dataplane",
+						// Built-in subcommand to check Envoy health
+						"-check-proxy-health",
+					},
+				},
+			},
+			InitialDelaySeconds: w.getSidecarProbeCheckInitialDelaySeconds(pod),
+			PeriodSeconds:       w.getSidecarProbePeriodSeconds(pod),
+			FailureThreshold:    w.getSidecarProbeFailureThreshold(pod),
+			TimeoutSeconds:      w.getSidecarProbeTimeoutSeconds(pod),
+		}
+	}
 	return container, nil
 }
 
@@ -615,6 +641,17 @@ func (w *MeshWebhook) getLivenessFailureSeconds(pod corev1.Pod) int32 {
 	return 0
 }
 
+func (w *MeshWebhook) getSidecarProbeCheckInitialDelaySeconds(pod corev1.Pod) int32 {
+	seconds := w.DefaultSidecarProbeCheckInitialDelaySeconds
+	if v, ok := pod.Annotations[constants.AnnotationSidecarInitialProbeCheckDelaySeconds]; ok {
+		seconds, _ = strconv.Atoi(v)
+	}
+	if seconds > 0 {
+		return int32(seconds)
+	}
+	return 0
+}
+
 // getMetricsPorts creates container ports for exposing services such as prometheus.
 // Prometheus in particular needs a named port for use with the operator.
 // https://github.com/hashicorp/consul-k8s/pull/1440
@@ -648,3 +685,34 @@ func (w *MeshWebhook) getMetricsPorts(pod corev1.Pod) ([]corev1.ContainerPort, e
 		},
 	}, nil
 }
+
+func (w *MeshWebhook) getSidecarProbePeriodSeconds(pod corev1.Pod) int32 {
+	seconds := w.DefaultSidecarProbePeriodSeconds
+	if v, ok := pod.Annotations[constants.AnnotationSidecarProbePeriodSeconds]; ok {
+		seconds, _ = strconv.Atoi(v)
+	}
+	if seconds > 0 {
+		return int32(seconds)
+	}
+	return 0
+}
+func (w *MeshWebhook) getSidecarProbeFailureThreshold(pod corev1.Pod) int32 {
+	threshold := w.DefaultSidecarProbeFailureThreshold
+	if v, ok := pod.Annotations[constants.AnnotationSidecarProbeFailureThreshold]; ok {
+		threshold, _ = strconv.Atoi(v)
+	}
+	if threshold > 0 {
+		return int32(threshold)
+	}
+	return 0
+}
+func (w *MeshWebhook) getSidecarProbeTimeoutSeconds(pod corev1.Pod) int32 {
+	seconds := w.DefaultSidecarProbeCheckTimeoutSeconds
+	if v, ok := pod.Annotations[constants.AnnotationSidecarProbeCheckTimeoutSeconds]; ok {
+		seconds, _ = strconv.Atoi(v)
+	}
+	if seconds > 0 {
+		return int32(seconds)
+	}
+	return 0
+}