WIP: Add Azure Linux 3 prow template for testing

templates/test/ci/prow-azl3/kustomization.yaml

@@ -0,0 +1,36 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+- ../../../flavors/default
+- ../../../addons/cluster-api-helm/calico.yaml
+- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml
+- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml
+- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml
+patches:
+- path: ../patches/tags.yaml
+- path: ../patches/mhc.yaml
+- path: ../patches/controller-manager.yaml
+- path: ../patches/uami-md-0.yaml
+- path: ../patches/uami-control-plane.yaml
+- path: ../patches/cluster-label-calico.yaml
+- path: ../patches/cluster-label-cloud-provider-azure.yaml
+- path: patches/controller-manager.yaml
+  target:
+    group: controlplane.cluster.x-k8s.io
+    kind: KubeadmControlPlane
+    name: .*-control-plane
+    version: v1beta1
+- path: patches/kubeadm-config-template-azl3.yaml
+  target:
+    group: bootstrap.cluster.x-k8s.io
+    kind: KubeadmConfigTemplate
+    name: .*-md-0
+    namespace: default
+    version: v1beta1
+- path: patches/azuremachinetemplate-azl3-image.yaml
+- path: patches/cloud-provider-azure-cacertdir.yaml
+- path: patches/cloud-provider-azure-ci-cacertdir.yaml
+
+sortOptions:
+  order: fifo

templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml

@@ -0,0 +1,25 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      image:
+        computeGallery:
+          gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019
+          name: capi-azurelinux-3
+          version: ${AZL3_VERSION:="1.33.2"}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      image:
+        computeGallery:
+          gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019
+          name: capi-azurelinux-3
+          version: ${AZL3_VERSION:="1.33.2"}

templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml

@@ -0,0 +1,12 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart
+spec:
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      caCertDir: "/etc/pki/tls/certs"
+      clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
+      logVerbosity: 4

templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml

@@ -0,0 +1,23 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart-ci
+spec:
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      caCertDir: "/etc/pki/tls/certs"
+      cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
+      cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
+      clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
+      imageName: "${CCM_IMAGE_NAME:-""}"
+      imageRepository: "${IMAGE_REGISTRY:-""}"
+      imageTag: "${IMAGE_TAG_CCM:-""}"
+      logVerbosity: ${CCM_LOG_VERBOSITY:-4}
+      replicas: ${CCM_COUNT:-1}
+      enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
+    cloudNodeManager:
+      imageName: "${CNM_IMAGE_NAME:-""}"
+      imageRepository: "${IMAGE_REGISTRY:-""}"
+      imageTag: "${IMAGE_TAG_CNM:-""}"

templates/test/ci/prow-azl3/patches/controller-manager.yaml

@@ -0,0 +1,37 @@
+- op: add
+  path: /spec/kubeadmConfigSpec/files/0
+  value:
+    content: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+
+      # Install ca-certificates packages for Azure Linux
+      tdnf install -y ca-certificates ca-certificates-legacy
+      update-ca-trust
+
+      # Follow Azure Linux 3 docs exactly - completely permissive for debugging
+      # Change default policy to ACCEPT (as recommended by AZL3 docs)
+      iptables -P INPUT ACCEPT
+      iptables -P FORWARD ACCEPT
+      iptables -P OUTPUT ACCEPT
+
+      ip6tables -P INPUT ACCEPT
+      ip6tables -P FORWARD ACCEPT
+      ip6tables -P OUTPUT ACCEPT
+
+      # Flush any rules which would filter packets
+      iptables -F
+      ip6tables -F
+
+      iptables-save > /etc/systemd/scripts/ip4save
+      ip6tables-save > /etc/systemd/scripts/ip6save
+    path: /tmp/azl3-setup.sh
+    owner: "root:root"
+    permissions: "0744"
+- op: add
+  path: /spec/kubeadmConfigSpec/preKubeadmCommands/0
+  value:
+    bash -c /tmp/azl3-setup.sh

templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml

@@ -0,0 +1,17 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      disableVMBootstrapExtension: true
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      disableVMBootstrapExtension: true

templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml

@@ -0,0 +1,37 @@
+- op: add
+  path: /spec/template/spec/files/0
+  value:
+    content: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+
+      # Install ca-certificates packages for Azure Linux
+      tdnf install -y ca-certificates ca-certificates-legacy
+      update-ca-trust
+
+      # Follow Azure Linux 3 docs exactly - completely permissive for debugging
+      # Change default policy to ACCEPT (as recommended by AZL3 docs)
+      iptables -P INPUT ACCEPT
+      iptables -P FORWARD ACCEPT
+      iptables -P OUTPUT ACCEPT
+
+      ip6tables -P INPUT ACCEPT
+      ip6tables -P FORWARD ACCEPT
+      ip6tables -P OUTPUT ACCEPT
+
+      # Flush any rules which would filter packets
+      iptables -F
+      ip6tables -F
+
+      iptables-save > /etc/systemd/scripts/ip4save
+      ip6tables-save > /etc/systemd/scripts/ip6save
+    path: /tmp/azl3-setup.sh
+    owner: "root:root"
+    permissions: "0744"
+- op: add
+  path: /spec/template/spec/preKubeadmCommands/0
+  value:
+    bash -c /tmp/azl3-setup.sh

templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml

@@ -0,0 +1,19 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      image:
+        marketplace: null
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      image:
+        marketplace: null

templates/test/ci/prow-ci-version-azl3/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+- ../prow-ci-version
+patches:
+- path: ../prow-azl3/patches/controller-manager.yaml
+  target:
+    group: controlplane.cluster.x-k8s.io
+    kind: KubeadmControlPlane
+    name: .*-control-plane
+    version: v1beta1
+- path: ../prow-azl3/patches/kubeadm-config-template-azl3.yaml
+  target:
+    group: bootstrap.cluster.x-k8s.io
+    kind: KubeadmConfigTemplate
+    name: .*-md-0
+    namespace: default
+    version: v1beta1
+- path: ../prow-azl3/patches/azuremachinetemplate-azl3-image.yaml
+- path: ../prow-azl3/patches/remove-marketplace-image.yaml
+- path: ../prow-azl3/patches/cloud-provider-azure-cacertdir.yaml
+- path: ../prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml
+
+sortOptions:
+  order: fifo

test/e2e/azure_logcollector.go

@@ -443,15 +443,15 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ...
 		),
 		execToPathFn(
 			"cloud-init.log",
-			"cat", "/var/log/cloud-init.log",
+			"sudo", "sh", "-c", "if [ -f /var/log/cloud-init.log ]; then sudo cat /var/log/cloud-init.log; else echo 'cloud-init.log not found'; fi",
 		),
 		execToPathFn(
 			"cloud-init-output.log",
-			"cat", "/var/log/cloud-init-output.log",
+			"sudo", "sh", "-c", "echo 'Waiting for cloud-init to complete before collecting output log...' && cloud-init status --wait && echo 'Cloud-init completed, collecting output log...' && if [ -f /var/log/cloud-init-output.log ]; then echo 'Found cloud-init-output.log, reading contents:' && sudo cat /var/log/cloud-init-output.log; else echo 'cloud-init-output.log not found after cloud-init completion'; fi",
 		),
 		execToPathFn(
 			"sentinel-file-dir.txt",
-			"ls", "/run/cluster-api/",
+			"ls", "-la", "/run/cluster-api/",
 		),
 		execToPathFn(
 			"cni.log",

test/e2e/azure_test.go

@@ -198,12 +198,15 @@ var _ = Describe("Workload cluster creation", func() {
 
 	Context("Creating a highly available cluster [REQUIRED]", func() {
 		It("With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes", func() {
+			Expect(os.Setenv("KUBERNETES_VERSION", "v1.33.2")).To(Succeed())
+			Expect(os.Setenv("SKIP_CLEANUP", "true")).To(Succeed())
 			clusterName = getClusterName(clusterNamePrefix, "ha")
 
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
 				specName,
 				withNamespace(namespace.Name),
 				withClusterName(clusterName),
+				withFlavor("azl3"),
 				withControlPlaneMachineCount(3),
 				withWorkerMachineCount(2),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
@@ -462,7 +465,7 @@ var _ = Describe("Workload cluster creation", func() {
 				withFlavor("ipv6"),
 				withNamespace(namespace.Name),
 				withClusterName(clusterName),
-				withControlPlaneMachineCount(3),
+				withControlPlaneMachineCount(1),
 				withWorkerMachineCount(1),
 				withControlPlaneInterval(specName, "wait-control-plane-ha"),
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
@@ -1292,5 +1295,91 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
+	Context("Creating a highly-available cluster with Azure Linux 3 [OPTIONAL]", func() {
+		It("with three controlplane node and two worker nodes", func() {
+			clusterName = getClusterName(clusterNamePrefix, "azl3")
+			Expect(os.Setenv("AZL3_VERSION", "${KUBERNETES_VERSION}")).To(Succeed())
+
+			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
+				specName,
+				withNamespace(namespace.Name),
+				withClusterName(clusterName),
+				withFlavor("azl3"),
+				withControlPlaneMachineCount(3),
+				withWorkerMachineCount(2),
+				withControlPlaneInterval(specName, "wait-control-plane-ha"),
+				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
+					WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+				}),
+				withPostMachinesProvisioned(func() {
+					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
+						return DaemonsetsSpecInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
+				}),
+			), result)
+
+			By("Verifying expected VM extensions are present on the node", func() {
+				AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput {
+					return AzureVMExtensionsSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+					}
+				})
+			})
+
+			By("Verifying security rules are deleted on azure side", func() {
+				AzureSecurityGroupsSpec(ctx, func() AzureSecurityGroupsSpecInput {
+					return AzureSecurityGroupsSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+						Cluster:               result.Cluster,
+						WaitForUpdate:         e2eConfig.GetIntervals(specName, "wait-nsg-update"),
+					}
+				})
+			})
+
+			By("Validating failure domains", func() {
+				AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput {
+					return AzureFailureDomainsSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Cluster:               result.Cluster,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+					}
+				})
+			})
+
+			By("Creating an accessible load balancer", func() {
+				AzureLBSpec(ctx, func() AzureLBSpecInput {
+					return AzureLBSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+						SkipCleanup:           skipCleanup,
+					}
+				})
+			})
+
+			By("Validating network policies", func() {
+				AzureNetPolSpec(ctx, func() AzureNetPolSpecInput {
+					return AzureNetPolSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+						SkipCleanup:           skipCleanup,
+					}
+				})
+			})
+
+			By("PASSED!")
+		})
+	})
+
 	// TODO: add a same test as above for a windows cluster
 })

test/e2e/config/azure-dev.yaml

@@ -148,6 +148,8 @@ providers:
         targetName: "cluster-template-conformance-presubmit-artifacts-dra.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-dra.yaml"
         targetName: "cluster-template-conformance-ci-artifacts-dra.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts-azl3.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml"
         targetName: "cluster-template-machine-pool-flex.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml"
@@ -184,6 +186,8 @@ providers:
         targetName: "cluster-template-apiserver-ilb.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml"
         targetName: "cluster-template-apiserver-ilb-custom-images.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azl3.yaml"
+        targetName: "cluster-template-azl3.yaml"
       replacements:
       - old: "--v=0"
         new: "--v=2"