Draft: Inject consul-dataplane as native sidecar

.changelog/4479.txt

@@ -0,0 +1,7 @@
+```release-note:feature
+control-plane: Allow to inject consul-dataplane as native sidecar.
+```
+
+```release-note:feature
+helm: allow to set the flag to inject consul-dataplane as native sidecar.
+```

charts/consul/templates/connect-inject-deployment.yaml

@@ -261,6 +261,11 @@ spec:
                 -default-sidecar-proxy-lifecycle-graceful-startup-path="{{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulStartupPath }}" \
                 -default-sidecar-proxy-startup-failure-seconds={{ .Values.connectInject.sidecarProxy.defaultStartupFailureSeconds }} \
                 -default-sidecar-proxy-liveness-failure-seconds={{ .Values.connectInject.sidecarProxy.defaultLivenessFailureSeconds }} \
+                {{- if .Values.connectInject.sidecarProxy.nativeSidecarsEnabled }}
+                -native-sidecars-enabled=true \
+                {{- else }}
+                -native-sidecars-enabled=false \
+                {{- end }}
                 {{- if .Values.connectInject.initContainer }}
                 {{- $initResources := .Values.connectInject.initContainer.resources }}
                 {{- if not (kindIs "invalid" $initResources.limits.memory) }}

charts/consul/values.yaml

@@ -3001,6 +3001,9 @@ connectInject:
     # A value of zero disables the probe.
     defaultLivenessFailureSeconds: 0
 
+    # Use native Sidecars to inject consul dataplane
+    nativeSidecarsEnabled: false
+
   # The resource settings for the Connect injected init container. If null, the resources
   # won't be set for the initContainer. The defaults are optimized for developer instances of
   # Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times.

control-plane/connect-inject/constants/annotations_and_labels.go

@@ -123,6 +123,9 @@ const (
 	AnnotationSidecarProxyLifecycleGracefulShutdownPath         = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path"
 	AnnotationSidecarProxyLifecycleGracefulStartupPath          = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-startup-path"
 
+	// annotation for injecting consul dataplane as native sidecar.
+	AnnotationNativeSidecarsEnabled = "consul.hashicorp.com/native-sidecars-enabled"
+
 	// annotations for sidecar volumes.
 	AnnotationConsulSidecarUserVolume      = "consul.hashicorp.com/consul-sidecar-user-volume"
 	AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount"

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -176,6 +176,11 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount)
 	}
 
+	if w.NativeSidecarsEnabled {
+		restartPolicy := corev1.ContainerRestartPolicyAlways
+		container.RestartPolicy = &restartPolicy
+	}
+
 	if useProxyHealthCheck(pod) {
 		// Configure the Readiness Address for the proxy's health check to be the Pod IP.
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -610,6 +615,20 @@ func (w *MeshWebhook) getLivenessFailureSeconds(pod corev1.Pod) int32 {
 	return 0
 }
 
+// getNativeSidecarsEnabled returns whether native sidecars is enabled either via the default value in the meshWebhook, or if it's been
+// overridden via the annotation.
+func (w *MeshWebhook) getNativeSidecarsEnabled(pod corev1.Pod) bool {
+	enabled := w.NativeSidecarsEnabled
+	if v, ok := pod.Annotations[constants.AnnotationNativeSidecarsEnabled]; ok {
+		nativeSidecarsEnabled, err := strconv.ParseBool(v)
+		if err != nil {
+			return enabled
+		}
+		return nativeSidecarsEnabled
+	}
+	return enabled
+}
+
 // getMetricsPorts creates container ports for exposing services such as prometheus.
 // Prometheus in particular needs a named port for use with the operator.
 // https://github.com/hashicorp/consul-k8s/pull/1440

control-plane/connect-inject/webhook/mesh_webhook.go

@@ -164,6 +164,9 @@ type MeshWebhook struct {
 	// configuration should come from the default flags or annotations. The meshWebhook uses this to configure container sidecar proxy args.
 	LifecycleConfig lifecycle.Config
 
+	// Use native Sidecars to inject consul dataplane.
+	NativeSidecarsEnabled bool
+
 	// Default Envoy concurrency flag, this is the number of worker threads to be used by the proxy.
 	DefaultEnvoyProxyConcurrency int
 
@@ -322,12 +325,17 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi
 			w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name)
 			return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err))
 		}
-		//Append the Envoy sidecar before the application container only if lifecycle enabled.
 
-		if lifecycleEnabled && ok == nil {
-			pod.Spec.Containers = append([]corev1.Container{envoySidecar}, pod.Spec.Containers...)
+		// Append the Envoy sidecar before the application container only if lifecycle enabled.
+		// Use Kubernetes Native Sidecars if nativeSidecars enabled
+		if w.getNativeSidecarsEnabled(pod) {
+			pod.Spec.InitContainers = append(pod.Spec.InitContainers, envoySidecar)
 		} else {
-			pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
+			if lifecycleEnabled && ok == nil {
+				pod.Spec.Containers = append([]corev1.Container{envoySidecar}, pod.Spec.Containers...)
+			} else {
+				pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
+			}
 		}
 
 	} else {
@@ -405,11 +413,15 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi
 			}
 			// If Lifecycle is enabled, add to the list of sidecar containers to be added
 			// to pod containers at the end in order to preserve relative ordering.
-			if lifecycleEnabled {
-				sidecarContainers = append(sidecarContainers, envoySidecar)
+			if w.getNativeSidecarsEnabled(pod) {
+				pod.Spec.InitContainers = append(pod.Spec.InitContainers, envoySidecar)
 			} else {
-				pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
+				if lifecycleEnabled {
+					sidecarContainers = append(sidecarContainers, envoySidecar)
+				} else {
+					pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar)
 
+				}
 			}
 
 		}

control-plane/connect-inject/webhook/mesh_webhook_test.go

@@ -1153,6 +1153,107 @@ func TestHandlerHandle(t *testing.T) {
 				// Note: no DNS policy/config additions.
 			},
 		},
+
+		{
+			"empty pod basic with native sidecars",
+			MeshWebhook{
+				Log:                   logrtest.New(t),
+				AllowK8sNamespacesSet: mapset.NewSetWith("*"),
+				DenyK8sNamespacesSet:  mapset.NewSet(),
+				decoder:               decoder,
+				Clientset:             defaultTestClientWithNamespace(),
+				NativeSidecarsEnabled: true,
+			},
+			admission.Request{
+				AdmissionRequest: admissionv1.AdmissionRequest{
+					Namespace: namespaces.DefaultNamespace,
+					Object: encodeRaw(t, &corev1.Pod{
+						Spec: basicSpec,
+					}),
+				},
+			},
+			"",
+			[]jsonpatch.Operation{
+				{
+					Operation: "add",
+					Path:      "/metadata/labels",
+				},
+				{
+					Operation: "add",
+					Path:      "/metadata/annotations",
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/volumes",
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/initContainers",
+				},
+			},
+		},
+
+		{
+			"empty pod basic with native sidecars annotation",
+			MeshWebhook{
+				Log:                   logrtest.New(t),
+				AllowK8sNamespacesSet: mapset.NewSetWith("*"),
+				DenyK8sNamespacesSet:  mapset.NewSet(),
+				decoder:               decoder,
+				Clientset:             defaultTestClientWithNamespace(),
+			},
+			admission.Request{
+				AdmissionRequest: admissionv1.AdmissionRequest{
+					Namespace: namespaces.DefaultNamespace,
+					Object: encodeRaw(t, &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{},
+							Annotations: map[string]string{
+								constants.AnnotationNativeSidecarsEnabled: "true",
+							},
+						},
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "web",
+								},
+							},
+						},
+					}),
+				},
+			},
+			"",
+			[]jsonpatch.Operation{
+				{
+					Operation: "add",
+					Path:      "/metadata/labels",
+				},
+				{
+					Operation: "add",
+					Path:      "/metadata/annotations/" + escapeJSONPointer(constants.KeyInjectStatus),
+				},
+				{
+					Operation: "add",
+					Path:      "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod),
+				},
+				{
+					Operation: "add",
+					Path:      "/metadata/annotations/" + escapeJSONPointer(constants.LegacyAnnotationConsulK8sVersion),
+				},
+				{
+					Operation: "add",
+					Path:      "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/volumes",
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/initContainers",
+				},
+			},
+		},
 	}
 
 	for _, tt := range cases {

control-plane/subcommand/inject-connect/command.go

@@ -94,6 +94,9 @@ type Command struct {
 	flagDefaultSidecarProxyStartupFailureSeconds  int
 	flagDefaultSidecarProxyLivenessFailureSeconds int
 
+	// native sidecars
+	flagNativeSidecarsEnabled bool
+
 	// Metrics settings.
 	flagDefaultEnableMetrics        bool
 	flagEnableGatewayMetrics        bool
@@ -257,6 +260,10 @@ func (c *Command) init() {
 	c.flagSet.IntVar(&c.flagDefaultSidecarProxyStartupFailureSeconds, "default-sidecar-proxy-startup-failure-seconds", 0, "Default number of seconds for the k8s startup probe to fail before the proxy container is restarted. Zero disables the probe.")
 	c.flagSet.IntVar(&c.flagDefaultSidecarProxyLivenessFailureSeconds, "default-sidecar-proxy-liveness-failure-seconds", 0, "Default number of seconds for the k8s liveness probe to fail before the proxy container is restarted. Zero disables the probe.")
 
+	// Native sidecars
+	c.flagSet.BoolVar(&c.flagNativeSidecarsEnabled, "enable-native-sidecars", false,
+		"Enables Consul dataplane injection using Kubernetes native Sidecards.")
+
 	// Metrics setting flags.
 	c.flagSet.BoolVar(&c.flagDefaultEnableMetrics, "default-enable-metrics", false, "Default for enabling connect service metrics.")
 	c.flagSet.BoolVar(&c.flagEnableGatewayMetrics, "enable-gateway-metrics", false, "Allows enabling Consul gateway metrics.")

control-plane/subcommand/inject-connect/v1controllers.go

@@ -385,6 +385,7 @@ func (c *Command) configureControllers(ctx context.Context, mgr manager.Manager,
 		DefaultSidecarProxyStartupFailureSeconds:  c.flagDefaultSidecarProxyStartupFailureSeconds,
 		DefaultSidecarProxyLivenessFailureSeconds: c.flagDefaultSidecarProxyLivenessFailureSeconds,
 		LifecycleConfig:                           lifecycleConfig,
+		NativeSidecarsEnabled:                     c.flagNativeSidecarsEnabled,
 		MetricsConfig:                             metricsConfig,
 		InitContainerResources:                    c.initContainerResources,
 		ConsulPartition:                           c.consul.Partition,