[Tilt] Drop SUBSCRIPTION_TYPE, do not peer vnets for all flavors and update vnet peering logic (#5587)

* remove setting SUBSCRIPTION_TYPE

- update resource deletion commands
- update docs

* update docs

Author: nawazkh

Tiltfile

@@ -97,7 +97,7 @@ if "aks_as_mgmt_settings" in settings and os_arch != "amd64":
     YELLOW = "\033[1;33m"
     RESET = "\033[0m"
     print("\n" + YELLOW + "ARCHITECTURE OVERRIDE: Using AKS as management cluster requires CAPZ to be built for amd64 architecture." + RESET)
-    print(YELLOW + "Ignoring local GOARCH output and forcing CAPZ's os_arch to amd64" + RESET + "\n")
+    print(YELLOW + "Ignoring local GOARCH=" + os_arch + " and building CAPZ images for os_arch=amd64" + RESET + "\n")
     os_arch = "amd64"
 
 # deploy CAPI
@@ -365,7 +365,7 @@ def flavors():
 
     delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false;"
 
-    if "aks_as_mgmt_settings" in settings and os.getenv("SUBSCRIPTION_TYPE", "") == "corporate":
+    if "aks_as_mgmt_settings" in settings:
         delete_all_workload_clusters += clear_aks_vnet_peerings()
 
     local_resource(
@@ -445,7 +445,7 @@ def deploy_worker_templates(template, substitutions):
     # Flavor command is built from here
     flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); "
 
-    if "aks_as_mgmt_settings" in settings and os.getenv("SUBSCRIPTION_TYPE", "") == "corporate" and "aks" not in flavor_name:
+    if "aks_as_mgmt_settings" in settings and needs_vnet_peering(flavor_name):
         apiserver_lb_private_ip = os.getenv("AZURE_INTERNAL_LB_PRIVATE_IP", "")
         if "windows-apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
             flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"40.0.11.100\"; "
@@ -454,7 +454,7 @@ def deploy_worker_templates(template, substitutions):
 
     flavor_cmd += "export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; "
 
-    if "aks_as_mgmt_settings" in settings and os.getenv("SUBSCRIPTION_TYPE", "") == "corporate" and "aks" not in flavor_name:
+    if "aks_as_mgmt_settings" in settings and needs_vnet_peering(flavor_name):
         flavor_cmd += peer_vnets()
 
     # wait for kubeconfig to be available
@@ -478,7 +478,7 @@ def deploy_worker_templates(template, substitutions):
         """ + kubectl_cmd + """ --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml |         sed 's/namespace: kube-system/namespace: calico-system/' |         """ + kubectl_cmd + """ --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -;
         """
 
-    if "aks_as_mgmt_settings" in settings and os.getenv("SUBSCRIPTION_TYPE", "") == "corporate" and "aks" not in flavor_name:
+    if "aks_as_mgmt_settings" in settings and needs_vnet_peering(flavor_name):
         flavor_cmd += create_private_dns_zone()
 
     flavor_cmd += get_addons(flavor_name)
@@ -627,13 +627,32 @@ def check_nodes_ready(flavor_name):
 
 def clear_aks_vnet_peerings():
     delete_peering_cmd = '''
-    echo "--------Clearing AKS MGMT VNETs Peerings--------";
-    az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180;
-    echo "VNet ${AKS_MGMT_VNET_NAME} found ";
 
-    PEERING_NAMES=$(az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --query "[].name" --output tsv);
-    for PEERING_NAME in ${PEERING_NAMES}; do echo "Deleting peering: ${PEERING_NAME}"; az network vnet peering delete --name ${PEERING_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME}; done;
-    echo "All VNETs Peerings deleted in ${AKS_MGMT_VNET_NAME}";
+    # Bail out early if the VNet itself never shows up.
+    if ! az network vnet wait \
+    --resource-group "${AKS_RESOURCE_GROUP}" \
+    --name "${AKS_MGMT_VNET_NAME}" \
+    --created --timeout 180; then
+        echo "VNet ${AKS_MGMT_VNET_NAME} not found in resource group ${AKS_RESOURCE_GROUP} after 180 seconds - bailing out"
+        exit 0
+    fi
+
+    az network vnet peering list \
+    --resource-group "$AKS_RESOURCE_GROUP" \
+    --vnet-name      "$AKS_MGMT_VNET_NAME" \
+    --query '[].name' -o tsv |
+    while IFS= read -r PEERING; do
+        [ -z "$PEERING" ] && continue
+
+        echo "Deleting peering: $PEERING"
+        az network vnet peering delete \
+            --name "$PEERING" \
+            --resource-group "$AKS_RESOURCE_GROUP" \
+            --vnet-name "$AKS_MGMT_VNET_NAME" \
+        || echo "Peering $PEERING already gone – skipping."
+    done
+
+    echo "Done"
     '''
 
     return delete_peering_cmd
@@ -739,6 +758,32 @@ def allow_tcp_udp_ports():
         allow_parallel = True,
     )
 
+def needs_vnet_peering(flavor_name):
+    """
+    Check if the flavor requires VNet peering configuration.
+
+    Args:
+        flavor_name (str): The name of the flavor to check
+
+    Returns:
+        bool: True if the flavor needs VNet peering, False otherwise
+
+    Flavors requiring VNet peering are:
+    - apiserver-ilb
+    - windows-apiserver-ilb
+    - aks
+    """
+    flavors_needing_vnet_peering = [
+        "apiserver-ilb",
+        "windows-apiserver-ilb",
+    ]
+
+    for f in flavors_needing_vnet_peering:
+        if f in flavor_name:
+            return True
+
+    return False
+
 ##############################
 # Actual work happens here
 ##############################
@@ -766,7 +811,7 @@ create_crs()
 
 flavors()
 
-if "aks_as_mgmt_settings" in settings and os.getenv("SUBSCRIPTION_TYPE", "") == "corporate":
+if "aks_as_mgmt_settings" in settings:
     allow_tcp_udp_ports()
 
 print("\n\n=== Active Tilt Configuration Settings ===")

docs/book/src/developers/tilt-with-aks-as-mgmt-ilb.md

@@ -1,11 +1,13 @@
 # Tilt with AKS as Management Cluster with Internal Load Balancer
 
 ## Introduction
+
 This guide is explaining how to set up and use Azure Kubernetes Service (AKS) as a management cluster for Cluster API Provider Azure (CAPZ) development using Tilt and an internal load balancer (ILB).
 
 While the default Tilt setup recommends using a KIND cluster as the management cluster for faster development and experimentation, this guide demonstrates using AKS as an alternative management cluster. We also cover additional steps for working with stricter network policies - particularly useful for organizations that need to maintain all cluster communications within their Azure Virtual Network (VNet) infrastructure with enhanced access controls.
 
 ### Who is this for?
+
 - Developers who want to use AKS as the management cluster for CAPZ development.
 - Developers working in environments with strict network security requirements.
 - Teams that need to keep all Kubernetes API traffic within Azure VNet
@@ -31,8 +33,10 @@ While the default Tilt setup recommends using a KIND cluster as the management c
 - If `tilt-settings.yaml` file exists in the root of your repo, clear out any values in `kustomize_settings` unless you want to use them instead of the values that will be set by running `make aks-create`.
 
 ### Managed Identity & Registry Setup
+
 1. Have a managed identity created from Azure Portal.
 2. Add the following lines to your shell config such as `~/.bashrc` or `~/.zshrc`
+
    ```shell
    export USER_IDENTITY="<user-assigned-managed-identity-name>"
    export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="<user-assigned-managed-identity-client-id>"
@@ -42,19 +46,24 @@ While the default Tilt setup recommends using a KIND cluster as the management c
    export AZURE_LOCATION="<azure-location-having-quota-for-B2s-and-D4s_v3-SKU>"
    export REGISTRY=<your-container-registry>
    ```
+
 3. Be sure to reload with `source ~/.bashrc` or `source ~/.zshrc` and then verify the correct env vars values return with `echo $AZURE_CLIENT_ID` and `echo $REGISTRY`.
 
 ## Steps to Use Tilt with AKS as the Management Cluster
 
-1. In tilt-settings.yaml, set subscription_type to "corporate" and remove any other env values unless you want to override env variables created by `make aks-create`. Example:
-   ```
-   .
-   .
-   .
-   kustomize_substitutions:
-   SUBSCRIPTION_TYPE: "corporate"
-   .
+1. Ensure that the tilt-settings.yaml in root of the repository looks like below
+
+   ```yaml
+      kustomize_substitutions: {}
+      allowed_contexts:
+      - "kind-capz"
+      container_args:
+         capz-controller-manager:
+            - "--v=4"
    ```
+
+      - Add env variables in `kustomize_substitutions` if you want the added env variables to take precedence over the env values exported by running `make aks-create`.
+      - Port over an variables set in `tilt-settings.json` to `tilt-settings.yaml`'s `kustomize_substitution:{}` and delete `tilt-settings.json` if present in your local.
 2. `make clean`
    - This make target does not need to be run every time. Run it to remove bin and kubeconfigs.
 3. `make generate`
@@ -96,7 +105,7 @@ Running an e2e test locally in a restricted environment calls for some workaroun
   2. Assign that managed identity a contributor role to your subscription
   3. Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY`, and `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to the user-assigned managed identity.
 
-#### Update prow template with apiserver ILB networking solution
+### Update prow template with apiserver ILB networking solution
 
 There are three sections of a prow template that need an update.
 
@@ -180,7 +189,7 @@ A sample kustomize command for updating a prow template via its kustomization.ya
         powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
 ```
 
-#### Peer Vnets of the management cluster and the workload cluster
+#### Peer VNets of the management cluster and the workload cluster
 
 Peering VNets, creating a private DNS zone with the FQDN of the workload cluster, and updating NSGs of the management and workload clusters can be achieved by running `scripts/peer-vnets.sh`.