Merge pull request #5304 from nawazkh/feature_flag_internal_ilb

Create and add InternalILB as a CAPZ feature flag

Author: k8s-ci-robot

Makefile

@@ -387,6 +387,7 @@ create-cluster: ## Create a workload development Kubernetes cluster on Azure in
 	EXP_MACHINE_POOL=true \
 	EXP_EDGEZONE=true \
 	EXP_ASO_API=true \
+	EXP_APISERVER_ILB=true \
 	$(MAKE) create-management-cluster \
 	create-workload-cluster
 
@@ -751,7 +752,7 @@ tilt-up: install-tools ## Start tilt and build kind cluster if needed.
 	@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
 		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
 	fi; \
-	CLUSTER_TOPOLOGY=true EXP_ASO_API=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true tilt up
+	CLUSTER_TOPOLOGY=true EXP_ASO_API=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true EXP_APISERVER_ILB=true tilt up
 
 .PHONY: delete-cluster
 delete-cluster: delete-workload-cluster  ## Deletes the example kind cluster "capz".

config/manager/manager.yaml

@@ -24,7 +24,7 @@ spec:
             - --leader-elect
             - "--diagnostics-address=${CAPZ_DIAGNOSTICS_ADDRESS:=:8443}"
             - "--insecure-diagnostics=${CAPZ_INSECURE_DIAGNOSTICS:=false}"
-            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}"
             - "--v=0"
           image: controller:latest
           imagePullPolicy: Always

docs/book/src/developers/development.md

@@ -21,6 +21,8 @@
     - [Tilt for dev in CAPZ](#tilt-for-dev-in-capz)
     - [Tilt for dev in both CAPZ and CAPI](#tilt-for-dev-in-both-capz-and-capi)
     - [Deploying a workload cluster](#deploying-a-workload-cluster)
+    - [Tilt for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic](#tilt-for-dev-using-internal-load-balancer-ilb-for-intra-cluster-node-apiserver-traffic)
+      - [Flavors for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic](#flavors-for-dev-using-internal-load-balancer-ilb-for-intra-cluster-node-apiserver-traffic)
     - [Viewing Telemetry](#viewing-telemetry)
     - [Debugging](#debugging)
   - [Manual Testing](#manual-testing)
@@ -151,6 +153,17 @@ development will span both CAPZ and CAPI, then follow the [CAPI and CAPZ instruc
 
 #### Tilt for dev in CAPZ
 
+<aside class="note warning">
+  <h2>Warning</h2>
+  <p>
+    To use an internal load balancer (ILB) intra-cluster node-apiserver traffic in your workload cluster, follow the
+    instructions in the
+    <a href="#tilt-for-dev-using-internal-load-balancer-ilb-for-intra-cluster-node-apiserver-traffic">
+      Tilt for Dev: Internal LB for Cluster VNet Communication
+    </a> section.
+  </p>
+</aside>
+
 If you want to develop in CAPZ and get a local development cluster working quickly, this is the path for you.
 
 Create a file named `tilt-settings.yaml` in the root of the CAPZ repository with the following contents:
@@ -247,6 +260,36 @@ make delete-workload-cluster
 
 > Check out the [self-managed](../self-managed/troubleshooting.md) and [managed](../managed/troubleshooting.md) troubleshooting guides for common errors you might run into.
 
+#### Tilt for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic
+
+This flow is for developers who want to leverage the internal load balancer for intra-cluster node-apiserver traffic.
+You can achieve this by setting the `EXP_APISERVER_ILB` environment variable to `true` in your shell (run `export EXP_APISERVER_ILB=true`) and then create the CAPZ management cluster.
+
+We also encourage you to use AKS cluster as your management cluster.
+
+Outline of the steps:
+- `make clean`
+- `make generate`
+- Set `REGISTRY` in your env. Preferably an Azure Container Registry.
+- Run `docker-build-all` to build all the images.
+- Run `make acr-login` to login to your ACR.
+- Run `docker-push-all` to push all the images to your ACR.
+- Run `make aks-create` to create an AKS cluster. _Notice the changes that get applied to the `tilt-settings.yaml` file._
+- Run `export EXP_APISERVER_ILB=true` in your shell.
+- Run `make tilt-up` to start Tilt.
+
+```
+TODO:
+1. Come up with a shorter make target to do all the above steps for MS Tenants.
+2. VNet peering should be exported out as a shell script for users to run.
+```
+
+##### Flavors for dev using internal load balancer (ILB) for intra-cluster node-apiserver traffic
+
+There are two flavors available for development in CAPZ for MSFT Tenant:
+- [apiserver-ilb](../../../../templates/cluster-template-apiserver-ilb.yaml): VM based default flavor that brings up native K8s clusters with Linux nodes.
+- [apiserver-ilb-windows](../../../../templates/cluster-template-windows-apiserver-ilb.yaml): VM based flavor that brings up native K8s clusters with Linux and Windows nodes.
+
 #### Viewing Telemetry
 
 The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD management cluster is

feature/feature.go

@@ -52,6 +52,12 @@ const (
 	// owner: @nojnhuh
 	// alpha: v1.15
 	ASOAPI featuregate.Feature = "ASOAPI"
+
+	// APIServerILB is a CAPZ feature gate to create an internal LB for the API Server of the workload cluster.
+	// Defaults to false.
+	// owner: @nawazkh
+	// alpha: v1.18
+	APIServerILB featuregate.Feature = "APIServerILB"
 )
 
 func init() {
@@ -66,4 +72,5 @@ var defaultCAPZFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	AKSResourceHealth: {Default: false, PreRelease: featuregate.Alpha},
 	EdgeZone:          {Default: false, PreRelease: featuregate.Alpha},
 	ASOAPI:            {Default: true, PreRelease: featuregate.Alpha},
+	APIServerILB:      {Default: false, PreRelease: featuregate.Alpha},
 }

hack/observability/opentelemetry/controller-manager-patch.yaml

@@ -12,5 +12,5 @@ spec:
             - "--diagnostics-address=:8080"
             - "--insecure-diagnostics"
             - "--leader-elect"
-            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}"
             - "--enable-tracing"