From a133b4dd3eb306b4539f3a0d1e7c5dc513bdd274 Mon Sep 17 00:00:00 2001 From: William Yao Date: Tue, 19 Aug 2025 09:45:25 -0700 Subject: [PATCH 01/19] Try azure linux 3 --- azure/defaults.go | 2 +- .../cluster-api-helm/cloud-provider-azure-ci.yaml | 1 + .../cluster-api-helm/cloud-provider-azure.yaml | 1 + ...-template-prow-apiserver-ilb-custom-images.yaml | 6 ++++++ .../ci/cluster-template-prow-apiserver-ilb.yaml | 8 +++++++- .../ci/cluster-template-prow-azure-cni-v1.yaml | 8 +++++++- .../ci/cluster-template-prow-ci-version-dra.yaml | 8 ++++++-- ...luster-template-prow-ci-version-dual-stack.yaml | 12 ++++++++++-- .../ci/cluster-template-prow-ci-version-ipv6.yaml | 12 ++++++++++-- ...cluster-template-prow-ci-version-md-and-mp.yaml | 12 ++++++++++-- .../test/ci/cluster-template-prow-ci-version.yaml | 12 ++++++++++-- ...-custom-builds-apiserver-ilb-custom-images.yaml | 0 .../test/ci/cluster-template-prow-custom-vnet.yaml | 8 +++++++- .../test/ci/cluster-template-prow-dual-stack.yaml | 8 +++++++- .../test/ci/cluster-template-prow-edgezone.yaml | 8 +++++++- .../ci/cluster-template-prow-flatcar-sysext.yaml | 6 ++++-- .../test/ci/cluster-template-prow-flatcar.yaml | 6 ++++-- templates/test/ci/cluster-template-prow-ipv6.yaml | 8 +++++++- ...ster-template-prow-machine-pool-ci-version.yaml | 8 ++++++-- .../cluster-template-prow-machine-pool-flex.yaml | 8 +++++++- .../ci/cluster-template-prow-machine-pool.yaml | 8 +++++++- .../test/ci/cluster-template-prow-nvidia-gpu.yaml | 8 +++++++- .../test/ci/cluster-template-prow-private.yaml | 8 ++++++-- templates/test/ci/cluster-template-prow-spot.yaml | 8 +++++++- .../test/ci/cluster-template-prow-topology.yaml | 2 ++ templates/test/ci/cluster-template-prow.yaml | 14 ++++++++++++-- templates/test/ci/patches/controller-manager.yaml | 5 +++++ .../ci/patches/kubeadm-config-template-azl3.yaml | 12 ++++++++++++ templates/test/ci/prow/kustomization.yaml | 1 + .../dev/cluster-template-custom-builds-dra.yaml | 2 ++ .../cluster-template-custom-builds-load-dra.yaml | 10 ++++++++++ .../dev/cluster-template-custom-builds-load.yaml | 10 ++++++++++ ...mplate-custom-builds-machine-pool-load-dra.yaml | 2 ++ ...r-template-custom-builds-machine-pool-load.yaml | 2 ++ ...luster-template-custom-builds-machine-pool.yaml | 2 ++ .../test/dev/cluster-template-custom-builds.yaml | 10 ++++++++++ .../v1beta1/cluster-template-kcp-remediation.yaml | 2 ++ .../v1beta1/cluster-template-kcp-scale-in.yaml | 2 ++ .../cluster-template-machine-and-machine-pool.yaml | 2 ++ .../v1beta1/cluster-template-machine-pool.yaml | 2 ++ .../v1beta1/cluster-template-md-remediation.yaml | 2 ++ .../v1beta1/cluster-template-node-drain.yaml | 2 ++ .../v1beta1/cluster-template-upgrades.yaml | 2 ++ .../v1beta1/cluster-template.yaml | 2 ++ 44 files changed, 231 insertions(+), 31 deletions(-) create mode 100644 templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml create mode 100644 templates/test/ci/patches/kubeadm-config-template-azl3.yaml diff --git a/azure/defaults.go b/azure/defaults.go index 02e5508fa5c..f5b28b95a41 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -50,7 +50,7 @@ const ( // DefaultPublicGalleryName is the default Azure compute gallery. DefaultPublicGalleryName = "ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019" // DefaultLinuxGalleryImageName is the default Linux community gallery image definition. - DefaultLinuxGalleryImageName = "capi-ubun2-2404" + DefaultLinuxGalleryImageName = "capi-azurelinux-3" // DefaultWindowsGalleryImageName is the default Windows community gallery image definition. DefaultWindowsGalleryImageName = "capi-win-2019-containerd" ) diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml index 04cfc66b24a..f04a8c6558d 100644 --- a/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml +++ b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml @@ -13,6 +13,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml index 7838783312e..e729586df93 100644 --- a/templates/addons/cluster-api-helm/cloud-provider-azure.yaml +++ b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml @@ -13,5 +13,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index 9af9dfcce2e..c8a583c82db 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -195,6 +195,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -467,6 +471,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -486,6 +491,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index b0db2aef17b..87f2915f356 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -121,7 +121,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -358,6 +362,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -377,6 +382,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index d512f1c6ba3..4f8e0005891 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -108,7 +108,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -275,6 +279,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -294,6 +299,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index ca736d04faf..2c2523819d6 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -227,8 +227,10 @@ spec: postKubeadmCommands: [] preKubeadmCommands: - bash -c /tmp/containerd-config.sh - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -805,6 +807,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -824,6 +827,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 04e4a9f445d..b57bf53e8bb 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -231,8 +231,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -449,6 +451,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -565,6 +571,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -584,6 +591,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index cfa5ecdd107..b089bf12833 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -238,8 +238,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -467,6 +469,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -583,6 +589,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -602,6 +609,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 0a873707764..91d65f322f6 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -210,8 +210,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -426,6 +428,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -815,6 +821,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -834,6 +841,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index a6371798ed7..dcc7470040c 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -210,8 +210,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -426,6 +428,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -815,6 +821,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -834,6 +841,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index a87c9da641b..6386380f522 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -114,7 +114,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -348,6 +352,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -367,6 +372,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index c6da620e0dd..bdcbf06eaed 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -128,7 +128,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -438,6 +442,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -457,6 +462,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 7c46b6bdc86..e464a243705 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -110,7 +110,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -331,6 +335,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -350,6 +355,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index d4c2258f876..76ce594fa7e 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -348,8 +348,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' - | jq -r .compute.name)/g" /etc/kubeadm.yml + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 8a479d69bb5..a62683c790a 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -121,8 +121,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' - | jq -r .compute.name)/g" /etc/kubeadm.yml + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index b8e8fe10ecb..e22f7129ab5 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -135,7 +135,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -458,6 +462,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -477,6 +482,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 7a9648382bc..a8cde1a6f12 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -209,8 +209,10 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - bash -c /tmp/oot-cred-provider.sh - - bash -c /tmp/kubeadm-bootstrap.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 5 machineTemplate: infrastructureRef: @@ -765,6 +767,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -784,6 +787,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index d135e419b2d..f305b98ab17 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -111,7 +111,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -498,6 +502,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -517,6 +522,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 4b1ad679642..ab2f43f13b4 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -111,7 +111,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -492,6 +496,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -511,6 +516,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 8ea79155209..09c6e463209 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -108,7 +108,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -322,6 +326,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -341,6 +346,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 20a7a0d8e1c..ea5f94ecef5 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -142,8 +142,10 @@ spec: ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io apiserver' >> /etc/hosts; fi preKubeadmCommands: - - if [ -f /tmp/kubeadm.yaml ] || [ -f /run/kubeadm/kubeadm.yaml ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io - apiserver' >> /etc/hosts; fi + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -373,6 +375,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -392,6 +395,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index dcd6b559e2c..d071d002f80 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -107,7 +107,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -344,6 +348,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -363,6 +368,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index b1ffd57246e..2ebd68f9095 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -189,6 +189,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -208,6 +209,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 6be8c8d38f6..6d5409c438c 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -111,7 +111,11 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust verbosity: 10 machineTemplate: infrastructureRef: @@ -219,7 +223,11 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -547,6 +555,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -566,6 +575,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index 007509ef572..7225617cd87 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -4,6 +4,11 @@ metadata: name: "${CLUSTER_NAME}-control-plane" spec: kubeadmConfigSpec: + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust clusterConfiguration: controllerManager: extraArgs: diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml new file mode 100644 index 00000000000..cee2713ed79 --- /dev/null +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -0,0 +1,12 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust \ No newline at end of file diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 81f7b28fb41..5ca27992f0d 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -50,6 +50,7 @@ patches: - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml - path: ../patches/uami-md-0.yaml +- path: ../patches/kubeadm-config-template-azl3.yaml configMapGenerator: - files: - windows-cni=../../../addons/windows/calico/calico.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-dra.yaml b/templates/test/dev/cluster-template-custom-builds-dra.yaml index e703f6574cb..4620ff522e5 100644 --- a/templates/test/dev/cluster-template-custom-builds-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-dra.yaml @@ -759,6 +759,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -778,6 +779,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index a255a7bc590..e8965ce5ea5 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -225,6 +225,10 @@ spec: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - bash -c /tmp/containerd-config.sh + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -410,6 +414,10 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/containerd-config.sh + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -825,6 +833,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -844,6 +853,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index dfb0194c462..f15b7f657ec 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -208,6 +208,10 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -382,6 +386,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -787,6 +795,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -806,6 +815,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml index ada9bba7833..7e69118c5ef 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml @@ -769,6 +769,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -788,6 +789,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml index 6d6dec5a232..116c7f70906 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml @@ -729,6 +729,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -748,6 +749,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 54dfc708b3f..9a14e8deaaf 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -719,6 +719,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -738,6 +739,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b1ad2a278f3..010ebbcd161 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -202,6 +202,10 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -376,6 +380,10 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - |- + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -781,6 +789,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -800,6 +809,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml index 778e4527583..53056b07df1 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml index 087423e72ee..73e18e8de23 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml index 26f72ca9b12..ac2ee9ff492 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml index 71b94a0dee5..950986fafca 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml index 59d5e557127..41a5dbf0f6e 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml index 629ed5f1960..6fea9c4ff75 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml index 01ceecbf9e8..faa913963a0 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml index 89c45b8c1b1..2a0d1c8c65e 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml @@ -103,6 +103,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -121,6 +122,7 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} From 13b10eeb818cad282308f696d51bb4e805aab623 Mon Sep 17 00:00:00 2001 From: William Yao Date: Tue, 19 Aug 2025 12:19:01 -0700 Subject: [PATCH 02/19] Try 1 cp node --- test/e2e/azure_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 99d35e868d9..c44256f165b 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -204,7 +204,7 @@ var _ = Describe("Workload cluster creation", func() { specName, withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(3), + withControlPlaneMachineCount(1), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ @@ -290,7 +290,7 @@ var _ = Describe("Workload cluster creation", func() { withFlavor("azure-cni-v1"), withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(3), + withControlPlaneMachineCount(1), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ @@ -462,7 +462,7 @@ var _ = Describe("Workload cluster creation", func() { withFlavor("ipv6"), withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(3), + withControlPlaneMachineCount(1), withWorkerMachineCount(1), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ From 854e88f3bcfdc9bc16d1dad3d7027f7cacbd939c Mon Sep 17 00:00:00 2001 From: William Yao Date: Tue, 19 Aug 2025 17:29:05 -0700 Subject: [PATCH 03/19] try messing with iptables --- ...late-prow-apiserver-ilb-custom-images.yaml | 45 +++++++++++ .../cluster-template-prow-apiserver-ilb.yaml | 45 +++++++++++ .../cluster-template-prow-azure-cni-v1.yaml | 45 +++++++++++ .../cluster-template-prow-ci-version-dra.yaml | 45 +++++++++++ ...r-template-prow-ci-version-dual-stack.yaml | 81 +++++++++++++++++++ ...cluster-template-prow-ci-version-ipv6.yaml | 81 +++++++++++++++++++ ...er-template-prow-ci-version-md-and-mp.yaml | 81 +++++++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 81 +++++++++++++++++++ .../ci/cluster-template-prow-custom-vnet.yaml | 45 +++++++++++ .../ci/cluster-template-prow-dual-stack.yaml | 45 +++++++++++ .../ci/cluster-template-prow-edgezone.yaml | 45 +++++++++++ .../cluster-template-prow-flatcar-sysext.yaml | 45 +++++++++++ .../ci/cluster-template-prow-flatcar.yaml | 45 +++++++++++ .../test/ci/cluster-template-prow-ipv6.yaml | 45 +++++++++++ ...template-prow-machine-pool-ci-version.yaml | 45 +++++++++++ ...uster-template-prow-machine-pool-flex.yaml | 45 +++++++++++ .../cluster-template-prow-machine-pool.yaml | 45 +++++++++++ .../ci/cluster-template-prow-nvidia-gpu.yaml | 45 +++++++++++ .../ci/cluster-template-prow-private.yaml | 45 +++++++++++ .../test/ci/cluster-template-prow-spot.yaml | 45 +++++++++++ templates/test/ci/cluster-template-prow.yaml | 81 +++++++++++++++++++ .../test/ci/patches/controller-manager.yaml | 45 +++++++++++ .../patches/kubeadm-config-template-azl3.yaml | 38 ++++++++- ...uster-template-custom-builds-load-dra.yaml | 81 +++++++++++++++++++ .../cluster-template-custom-builds-load.yaml | 81 +++++++++++++++++++ .../dev/cluster-template-custom-builds.yaml | 81 +++++++++++++++++++ 26 files changed, 1450 insertions(+), 1 deletion(-) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index c8a583c82db..cb4490743ae 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -199,6 +199,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index 87f2915f356..2e35a1c5ba7 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -126,6 +126,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 4f8e0005891..67a817397e5 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -113,6 +113,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 2c2523819d6..fe1604e6ecf 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -231,6 +231,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index b57bf53e8bb..d11b618039d 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -235,6 +235,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: @@ -455,6 +500,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index b089bf12833..cf3cedb0bf0 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -242,6 +242,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: @@ -473,6 +518,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 91d65f322f6..45b96f03764 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -214,6 +214,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: @@ -432,6 +477,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index dcc7470040c..d309defd733 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -214,6 +214,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: @@ -432,6 +477,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 6386380f522..4549012c48f 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -119,6 +119,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index bdcbf06eaed..61d0a25461a 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -133,6 +133,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index e464a243705..967767c8689 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -115,6 +115,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index 76ce594fa7e..1360fb57ddc 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -352,6 +352,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index a62683c790a..b3b34d8cd72 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -125,6 +125,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index e22f7129ab5..54b4457426a 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -140,6 +140,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index a8cde1a6f12..b6457d34e2c 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -213,6 +213,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index f305b98ab17..f9e8de5ba10 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -116,6 +116,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index ab2f43f13b4..32b8a02c8a5 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -116,6 +116,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 09c6e463209..04384e43c73 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -113,6 +113,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index ea5f94ecef5..d2d5075442e 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -146,6 +146,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index d071d002f80..6aed35dcdea 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -112,6 +112,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 6d5409c438c..1b4b5d0bd5c 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -116,6 +116,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: infrastructureRef: @@ -228,6 +273,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index 7225617cd87..a4e8d6d7dc9 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -9,6 +9,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save clusterConfiguration: controllerManager: extraArgs: diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index cee2713ed79..d7aaba53ad5 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -9,4 +9,40 @@ spec: - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust \ No newline at end of file + update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save \ No newline at end of file diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index e8965ce5ea5..8fae9edd20a 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -229,6 +229,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -418,6 +463,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index f15b7f657ec..82ae72db905 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -212,6 +212,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -390,6 +435,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 010ebbcd161..68550bbc95a 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -206,6 +206,51 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow Kubernetes API server (port 6443) + iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT + # Allow etcd (ports 2379-2380) + iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT + # Allow kube-scheduler (port 10259) + iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT + # Allow kube-controller-manager (port 10257) + iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -384,6 +429,42 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust + # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico + # Azure Linux 3 has default DROP policy, need to allow required traffic + # Allow loopback traffic + iptables -I INPUT 1 -i lo -j ACCEPT + iptables -I OUTPUT 1 -o lo -j ACCEPT + # Allow established and related connections + iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + # Allow all traffic from Azure metadata service + iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT + iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT + # Allow SSH (port 22) for management + iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT + # Allow kubelet API (port 10250) + iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT + # Allow Calico BGP (port 179) + iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT + # Allow Calico VXLAN (port 4789) + iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT + # Allow Calico Typha (port 5473) + iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT + # Allow NodePort services (30000-32767) + iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT + # Allow all outbound traffic (Kubernetes components need to communicate) + iptables -P OUTPUT ACCEPT + # Allow inter-node communication (adjust based on your subnet) + iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT + iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT + iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT + # Save the iptables rules for Azure Linux 3 + iptables-save > /etc/systemd/scripts/ip4save + # Also configure ip6tables for IPv6 + ip6tables -I INPUT 1 -i lo -j ACCEPT + ip6tables -I OUTPUT 1 -o lo -j ACCEPT + ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + ip6tables -P OUTPUT ACCEPT + ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- From 6f7d0e398f5e72ce357f16d3b143d67edb333c7e Mon Sep 17 00:00:00 2001 From: William Yao Date: Wed, 20 Aug 2025 09:32:45 -0700 Subject: [PATCH 04/19] Disable bootstrap extension --- templates/cluster-template-aad.yaml | 1 + templates/cluster-template-apiserver-ilb.yaml | 1 + templates/cluster-template-azure-bastion.yaml | 2 ++ templates/cluster-template-azure-cni-v1.yaml | 2 ++ templates/cluster-template-dual-stack.yaml | 1 + templates/cluster-template-edgezone.yaml | 2 ++ templates/cluster-template-ephemeral.yaml | 2 ++ templates/cluster-template-flatcar-sysext.yaml | 1 + templates/cluster-template-flatcar.yaml | 1 + templates/cluster-template-ipv6.yaml | 1 + templates/cluster-template-machinepool-windows.yaml | 1 + templates/cluster-template-machinepool.yaml | 1 + templates/cluster-template-nvidia-gpu.yaml | 1 + templates/cluster-template-private.yaml | 2 ++ templates/cluster-template-windows-apiserver-ilb.yaml | 1 + templates/cluster-template-windows.yaml | 2 ++ templates/cluster-template.yaml | 2 ++ templates/flavors/base/cluster-template.yaml | 1 + templates/flavors/default/machine-deployment.yaml | 1 + .../ci/cluster-template-prow-apiserver-ilb-custom-images.yaml | 1 + templates/test/ci/cluster-template-prow-apiserver-ilb.yaml | 1 + templates/test/ci/cluster-template-prow-azure-cni-v1.yaml | 2 ++ templates/test/ci/cluster-template-prow-ci-version-dra.yaml | 1 + .../test/ci/cluster-template-prow-ci-version-dual-stack.yaml | 2 ++ templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml | 2 ++ .../test/ci/cluster-template-prow-ci-version-md-and-mp.yaml | 2 ++ templates/test/ci/cluster-template-prow-ci-version.yaml | 2 ++ templates/test/ci/cluster-template-prow-custom-vnet.yaml | 2 ++ templates/test/ci/cluster-template-prow-dual-stack.yaml | 1 + templates/test/ci/cluster-template-prow-edgezone.yaml | 2 ++ templates/test/ci/cluster-template-prow-flatcar-sysext.yaml | 1 + templates/test/ci/cluster-template-prow-flatcar.yaml | 1 + templates/test/ci/cluster-template-prow-ipv6.yaml | 1 + .../test/ci/cluster-template-prow-machine-pool-ci-version.yaml | 1 + templates/test/ci/cluster-template-prow-machine-pool-flex.yaml | 1 + templates/test/ci/cluster-template-prow-machine-pool.yaml | 1 + templates/test/ci/cluster-template-prow-nvidia-gpu.yaml | 1 + templates/test/ci/cluster-template-prow-private.yaml | 2 ++ templates/test/ci/cluster-template-prow-spot.yaml | 2 ++ templates/test/ci/cluster-template-prow.yaml | 2 ++ templates/test/dev/cluster-template-custom-builds-dra.yaml | 1 + templates/test/dev/cluster-template-custom-builds-load-dra.yaml | 2 ++ templates/test/dev/cluster-template-custom-builds-load.yaml | 2 ++ .../cluster-template-custom-builds-machine-pool-load-dra.yaml | 1 + .../dev/cluster-template-custom-builds-machine-pool-load.yaml | 1 + .../test/dev/cluster-template-custom-builds-machine-pool.yaml | 1 + templates/test/dev/cluster-template-custom-builds.yaml | 2 ++ 47 files changed, 67 insertions(+) diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 24451adadfb..ae14e6047a1 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -126,6 +126,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-apiserver-ilb.yaml b/templates/cluster-template-apiserver-ilb.yaml index 6339dc4a392..af1a74a41d4 100644 --- a/templates/cluster-template-apiserver-ilb.yaml +++ b/templates/cluster-template-apiserver-ilb.yaml @@ -135,6 +135,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index f00edd6f71d..8d11b47abd0 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -123,6 +123,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -164,6 +165,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index 55e31104921..d6a66f70080 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -123,6 +123,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned networkInterfaces: - privateIPConfigs: 110 @@ -167,6 +168,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true networkInterfaces: - privateIPConfigs: 110 subnetName: node-subnet diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index c5aac3e9941..0381af0aa2c 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -144,6 +144,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index f06ebfd315b..a102377c944 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -124,6 +124,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -165,6 +166,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 7305b98a83e..e05f43ed523 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -121,6 +121,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: cachingType: ReadOnly @@ -165,6 +166,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: cachingType: ReadOnly diffDiskSettings: diff --git a/templates/cluster-template-flatcar-sysext.yaml b/templates/cluster-template-flatcar-sysext.yaml index 5bcebbf76a5..e2a2d58eecc 100644 --- a/templates/cluster-template-flatcar-sysext.yaml +++ b/templates/cluster-template-flatcar-sysext.yaml @@ -290,6 +290,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 2efd2a4d43e..86b87c7d641 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -136,6 +136,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index 0be25635b0e..09bc7230a1a 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -149,6 +149,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index d4582954b92..ae64fea8f05 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -125,6 +125,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index a7caeff7c25..e7255ef6161 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -121,6 +121,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 3cc10d76386..a3a73798141 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -121,6 +121,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 7dbd441dbd6..947c1d400ae 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -135,6 +135,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -176,6 +177,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-windows-apiserver-ilb.yaml b/templates/cluster-template-windows-apiserver-ilb.yaml index 30302ec00d4..08c5ee87771 100644 --- a/templates/cluster-template-windows-apiserver-ilb.yaml +++ b/templates/cluster-template-windows-apiserver-ilb.yaml @@ -139,6 +139,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 66c184ede74..8241951320a 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -125,6 +125,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -166,6 +167,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index b970c267af6..a0ab1ce73a6 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -121,6 +121,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -162,6 +163,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 055a57e247d..b83012e3803 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -109,6 +109,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} osDisk: osType: "Linux" diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index 9aaf668473f..216d6511c01 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -29,6 +29,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true vmSize: ${AZURE_NODE_MACHINE_TYPE} osDisk: osType: "Linux" diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index cb4490743ae..cd971fc6fc8 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -266,6 +266,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index 2e35a1c5ba7..a0173fecced 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -192,6 +192,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 67a817397e5..3f42032ffe1 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -179,6 +179,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned networkInterfaces: - privateIPConfigs: 110 @@ -223,6 +224,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true networkInterfaces: - privateIPConfigs: 110 subnetName: node-subnet diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index fe1604e6ecf..338ede5e5e8 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -297,6 +297,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index d11b618039d..e5260cbe941 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -303,6 +303,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -355,6 +356,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index cf3cedb0bf0..ed0c2126932 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -310,6 +310,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -362,6 +363,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 45b96f03764..cb85cb6d18c 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -282,6 +282,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -333,6 +334,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index d309defd733..bb0c0b42cf9 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -282,6 +282,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -333,6 +334,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 4549012c48f..3f18c7accd4 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -185,6 +185,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -228,6 +229,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 61d0a25461a..70d3b84ff1b 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -199,6 +199,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 967767c8689..beafb904d8c 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -183,6 +183,7 @@ spec: managedDisk: storageAccountType: StandardSSD_LRS nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -225,6 +226,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index 1360fb57ddc..d74a056ffe4 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -457,6 +457,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index b3b34d8cd72..7b97ef09c96 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -191,6 +191,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 54b4457426a..79511738cf6 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -206,6 +206,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index b6457d34e2c..6146e1a50a9 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -279,6 +279,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index f9e8de5ba10..566950b78d4 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -182,6 +182,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 32b8a02c8a5..4b7183a932d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -182,6 +182,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 04384e43c73..fe7cfcbb7b0 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -179,6 +179,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index d2d5075442e..2cd9f441f64 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -212,6 +212,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -253,6 +254,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 6aed35dcdea..88ddcf79857 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -178,6 +178,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -221,6 +222,7 @@ metadata: spec: template: spec: + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 1b4b5d0bd5c..ec0318e5d13 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -184,6 +184,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -229,6 +230,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/test/dev/cluster-template-custom-builds-dra.yaml b/templates/test/dev/cluster-template-custom-builds-dra.yaml index 4620ff522e5..ac9fe980730 100644 --- a/templates/test/dev/cluster-template-custom-builds-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-dra.yaml @@ -253,6 +253,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index 8fae9edd20a..3472339e575 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -299,6 +299,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -350,6 +351,7 @@ spec: spec: additionalTags: monitoring: dra + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 82ae72db905..25a9a518015 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -282,6 +282,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -333,6 +334,7 @@ spec: spec: additionalTags: monitoring: load + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml index 7e69118c5ef..590a86a6153 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml @@ -261,6 +261,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml index 116c7f70906..ec5cd52581b 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml @@ -243,6 +243,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 9a14e8deaaf..13a9ea82295 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -235,6 +235,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 68550bbc95a..a57a09c83a4 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -276,6 +276,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -327,6 +328,7 @@ spec: spec: additionalTags: monitoring: virtualmachine + disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: From b0aa34d1e9ca1b71e4cccff94c246c608ac29c3f Mon Sep 17 00:00:00 2001 From: William Yao Date: Wed, 20 Aug 2025 13:14:35 -0700 Subject: [PATCH 05/19] Revert cp machine count to 3 --- test/e2e/azure_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index c44256f165b..4cda5f5775b 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -290,7 +290,7 @@ var _ = Describe("Workload cluster creation", func() { withFlavor("azure-cni-v1"), withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(1), + withControlPlaneMachineCount(3), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ From 592399f2602da9facd75a1389fbbf3c83d85820d Mon Sep 17 00:00:00 2001 From: William Yao Date: Wed, 20 Aug 2025 15:26:15 -0700 Subject: [PATCH 06/19] Fix iptables --- ...late-prow-apiserver-ilb-custom-images.yaml | 49 ++-------- .../cluster-template-prow-apiserver-ilb.yaml | 49 ++-------- .../cluster-template-prow-azure-cni-v1.yaml | 49 ++-------- .../cluster-template-prow-ci-version-dra.yaml | 49 ++-------- ...r-template-prow-ci-version-dual-stack.yaml | 93 ++++--------------- ...cluster-template-prow-ci-version-ipv6.yaml | 93 ++++--------------- ...er-template-prow-ci-version-md-and-mp.yaml | 93 ++++--------------- .../ci/cluster-template-prow-ci-version.yaml | 93 ++++--------------- .../ci/cluster-template-prow-custom-vnet.yaml | 49 ++-------- .../ci/cluster-template-prow-dual-stack.yaml | 49 ++-------- .../ci/cluster-template-prow-edgezone.yaml | 49 ++-------- .../cluster-template-prow-flatcar-sysext.yaml | 49 ++-------- .../ci/cluster-template-prow-flatcar.yaml | 49 ++-------- .../test/ci/cluster-template-prow-ipv6.yaml | 49 ++-------- ...template-prow-machine-pool-ci-version.yaml | 49 ++-------- ...uster-template-prow-machine-pool-flex.yaml | 49 ++-------- .../cluster-template-prow-machine-pool.yaml | 49 ++-------- .../ci/cluster-template-prow-nvidia-gpu.yaml | 49 ++-------- .../ci/cluster-template-prow-private.yaml | 49 ++-------- .../test/ci/cluster-template-prow-spot.yaml | 49 ++-------- templates/test/ci/cluster-template-prow.yaml | 93 ++++--------------- .../test/ci/patches/controller-manager.yaml | 49 ++-------- .../patches/kubeadm-config-template-azl3.yaml | 44 ++------- ...uster-template-custom-builds-load-dra.yaml | 93 ++++--------------- .../cluster-template-custom-builds-load.yaml | 93 ++++--------------- .../dev/cluster-template-custom-builds.yaml | 93 ++++--------------- 26 files changed, 290 insertions(+), 1331 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index cd971fc6fc8..cb6fd9bbf88 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -200,49 +200,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index a0173fecced..467b9ff34c8 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -127,49 +127,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 3f42032ffe1..163505dadad 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -114,49 +114,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 338ede5e5e8..20c4b5a0d49 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -232,49 +232,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index e5260cbe941..6faf593a933 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -236,49 +236,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: @@ -498,45 +465,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index ed0c2126932..abdf3942414 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -243,49 +243,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: @@ -516,45 +483,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index cb85cb6d18c..18e9c9005b7 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -215,49 +215,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: @@ -475,45 +442,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index bb0c0b42cf9..5aa17b62d70 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -215,49 +215,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: @@ -475,45 +442,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 3f18c7accd4..6aa27669102 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -120,49 +120,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 70d3b84ff1b..86d0cfc742f 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -134,49 +134,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index beafb904d8c..a4a0fb7762b 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -116,49 +116,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index d74a056ffe4..6b6002f53df 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -353,49 +353,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 7b97ef09c96..7c9a0997356 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -126,49 +126,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 79511738cf6..26801a4be44 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -141,49 +141,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 6146e1a50a9..8f1f6d0ec05 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -214,49 +214,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 566950b78d4..8ebf1472efd 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -117,49 +117,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 4b7183a932d..5111215fbac 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -117,49 +117,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index fe7cfcbb7b0..73083a69725 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -114,49 +114,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 2cd9f441f64..b102ec336cf 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -147,49 +147,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 88ddcf79857..7a1f55df655 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -113,49 +113,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index ec0318e5d13..4b469a057ee 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -117,49 +117,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 machineTemplate: @@ -271,45 +238,21 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index a4e8d6d7dc9..b518b821abc 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -10,49 +10,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save clusterConfiguration: controllerManager: diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index d7aaba53ad5..302e5e0c02c 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -10,39 +10,15 @@ spec: # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - ip6tables-save > /etc/systemd/scripts/ip6save \ No newline at end of file + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index 3472339e575..6a0fe1d2854 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -230,49 +230,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh @@ -461,45 +428,21 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/containerd-config.sh - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 25a9a518015..785109debdf 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -213,49 +213,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh @@ -433,45 +400,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index a57a09c83a4..c4cbbde65a5 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -207,49 +207,16 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow Kubernetes API server (port 6443) - iptables -I INPUT 5 -p tcp --dport 6443 -j ACCEPT - # Allow etcd (ports 2379-2380) - iptables -I INPUT 6 -p tcp --dport 2379:2380 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 7 -p tcp --dport 10250 -j ACCEPT - # Allow kube-scheduler (port 10259) - iptables -I INPUT 8 -p tcp --dport 10259 -j ACCEPT - # Allow kube-controller-manager (port 10257) - iptables -I INPUT 9 -p tcp --dport 10257 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 10 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 11 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 12 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 13 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 14 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 15 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 16 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh @@ -427,45 +394,21 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - |- + - | # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Configure iptables for Azure Linux 3 - allow necessary traffic for Kubernetes/Calico - # Azure Linux 3 has default DROP policy, need to allow required traffic - # Allow loopback traffic - iptables -I INPUT 1 -i lo -j ACCEPT - iptables -I OUTPUT 1 -o lo -j ACCEPT - # Allow established and related connections - iptables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT - # Allow all traffic from Azure metadata service - iptables -I INPUT 3 -s 168.63.129.16 -j ACCEPT - iptables -I OUTPUT 3 -d 168.63.129.16 -j ACCEPT - # Allow SSH (port 22) for management - iptables -I INPUT 4 -p tcp --dport 22 -j ACCEPT - # Allow kubelet API (port 10250) - iptables -I INPUT 5 -p tcp --dport 10250 -j ACCEPT - # Allow Calico BGP (port 179) - iptables -I INPUT 6 -p tcp --dport 179 -j ACCEPT - # Allow Calico VXLAN (port 4789) - iptables -I INPUT 7 -p udp --dport 4789 -j ACCEPT - # Allow Calico Typha (port 5473) - iptables -I INPUT 8 -p tcp --dport 5473 -j ACCEPT - # Allow NodePort services (30000-32767) - iptables -I INPUT 9 -p tcp --dport 30000:32767 -j ACCEPT - # Allow all outbound traffic (Kubernetes components need to communicate) + + # Change default policy to ACCEPT + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - # Allow inter-node communication (adjust based on your subnet) - iptables -I INPUT 10 -s 10.0.0.0/8 -j ACCEPT - iptables -I INPUT 11 -s 172.16.0.0/12 -j ACCEPT - iptables -I INPUT 12 -s 192.168.0.0/16 -j ACCEPT - # Save the iptables rules for Azure Linux 3 - iptables-save > /etc/systemd/scripts/ip4save - # Also configure ip6tables for IPv6 - ip6tables -I INPUT 1 -i lo -j ACCEPT - ip6tables -I OUTPUT 1 -o lo -j ACCEPT - ip6tables -I INPUT 2 -m state --state ESTABLISHED,RELATED -j ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh From cde282bf0fd98b778bfff8534c9bc2e3544dc4d6 Mon Sep 17 00:00:00 2001 From: William Yao Date: Wed, 20 Aug 2025 16:54:44 -0700 Subject: [PATCH 07/19] Try more iptable configs --- ...late-prow-apiserver-ilb-custom-images.yaml | 24 +++++++++- .../cluster-template-prow-apiserver-ilb.yaml | 24 +++++++++- .../cluster-template-prow-azure-cni-v1.yaml | 24 +++++++++- .../cluster-template-prow-ci-version-dra.yaml | 24 +++++++++- ...r-template-prow-ci-version-dual-stack.yaml | 48 ++++++++++++++++++- ...cluster-template-prow-ci-version-ipv6.yaml | 48 ++++++++++++++++++- ...er-template-prow-ci-version-md-and-mp.yaml | 48 ++++++++++++++++++- .../ci/cluster-template-prow-ci-version.yaml | 48 ++++++++++++++++++- .../ci/cluster-template-prow-custom-vnet.yaml | 24 +++++++++- .../ci/cluster-template-prow-dual-stack.yaml | 24 +++++++++- .../ci/cluster-template-prow-edgezone.yaml | 24 +++++++++- .../cluster-template-prow-flatcar-sysext.yaml | 24 +++++++++- .../ci/cluster-template-prow-flatcar.yaml | 24 +++++++++- .../test/ci/cluster-template-prow-ipv6.yaml | 24 +++++++++- ...template-prow-machine-pool-ci-version.yaml | 24 +++++++++- ...uster-template-prow-machine-pool-flex.yaml | 24 +++++++++- .../cluster-template-prow-machine-pool.yaml | 24 +++++++++- .../ci/cluster-template-prow-nvidia-gpu.yaml | 24 +++++++++- .../ci/cluster-template-prow-private.yaml | 24 +++++++++- .../test/ci/cluster-template-prow-spot.yaml | 24 +++++++++- templates/test/ci/cluster-template-prow.yaml | 48 ++++++++++++++++++- .../test/ci/patches/controller-manager.yaml | 24 +++++++++- .../patches/kubeadm-config-template-azl3.yaml | 24 +++++++++- ...uster-template-custom-builds-load-dra.yaml | 48 ++++++++++++++++++- .../cluster-template-custom-builds-load.yaml | 48 ++++++++++++++++++- .../dev/cluster-template-custom-builds.yaml | 48 ++++++++++++++++++- 26 files changed, 782 insertions(+), 34 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index cb6fd9bbf88..7fdfc44983c 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -200,15 +200,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index 467b9ff34c8..73103749722 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -127,15 +127,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 163505dadad..3692b99c627 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -114,15 +114,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 20c4b5a0d49..156dadaccf6 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -232,15 +232,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 6faf593a933..13ec248a995 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -236,15 +236,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -470,15 +492,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index abdf3942414..bca83e482d7 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -243,15 +243,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -488,15 +510,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 18e9c9005b7..653aa6c4c1d 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -215,15 +215,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -447,15 +469,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 5aa17b62d70..531e4e55a41 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -215,15 +215,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -447,15 +469,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 6aa27669102..439e6ba7cbf 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -120,15 +120,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 86d0cfc742f..01d11b58484 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -134,15 +134,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index a4a0fb7762b..9b87c5b6f97 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -116,15 +116,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index 6b6002f53df..9855570294e 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -353,15 +353,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 7c9a0997356..6758a6562f0 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -126,15 +126,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 26801a4be44..f695662801c 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -141,15 +141,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 8f1f6d0ec05..ee45edf7169 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -214,15 +214,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 8ebf1472efd..feb9e6e7d82 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -117,15 +117,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 5111215fbac..06183fef377 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -117,15 +117,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 73083a69725..ae1b9e2f9f0 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -114,15 +114,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index b102ec336cf..759ac0f20f6 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -147,15 +147,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 7a1f55df655..3ed78f92b6a 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -113,15 +113,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 4b469a057ee..8e8a176074a 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -117,15 +117,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 @@ -243,15 +265,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save --- diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index b518b821abc..3cb4b77c613 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -10,15 +10,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save clusterConfiguration: diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index 302e5e0c02c..0e8243aee18 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -11,14 +11,36 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index 6a0fe1d2854..6fbecdd1014 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -230,15 +230,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -433,15 +455,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 785109debdf..2ea598c38dd 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -213,15 +213,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -405,15 +427,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index c4cbbde65a5..279ece19992 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -207,15 +207,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -399,15 +421,37 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Change default policy to ACCEPT + # Completely reset iptables to permissive state for etcd connectivity + iptables -F + iptables -X + iptables -t nat -F + iptables -t nat -X + iptables -t mangle -F + iptables -t mangle -X + iptables -t raw -F 2>/dev/null || true + iptables -t raw -X 2>/dev/null || true iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT + ip6tables -F + ip6tables -X + ip6tables -t nat -F + ip6tables -t nat -X + ip6tables -t mangle -F + ip6tables -t mangle -X + ip6tables -t raw -F 2>/dev/null || true + ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT + # Allow all etcd communication explicitly (for worker nodes that might become control planes) + iptables -A INPUT -p tcp --dport 2379 -j ACCEPT + iptables -A INPUT -p tcp --dport 2380 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh From 6f8f53ecc95deb6f93e193611fd924cefe04e493 Mon Sep 17 00:00:00 2001 From: William Yao Date: Thu, 21 Aug 2025 09:40:12 -0700 Subject: [PATCH 08/19] More iptable fixes --- ...late-prow-apiserver-ilb-custom-images.yaml | 8 ++++++++ .../cluster-template-prow-apiserver-ilb.yaml | 8 ++++++++ .../cluster-template-prow-azure-cni-v1.yaml | 8 ++++++++ .../cluster-template-prow-ci-version-dra.yaml | 8 ++++++++ ...r-template-prow-ci-version-dual-stack.yaml | 16 +++++++++++++++ ...cluster-template-prow-ci-version-ipv6.yaml | 16 +++++++++++++++ ...er-template-prow-ci-version-md-and-mp.yaml | 16 +++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 16 +++++++++++++++ .../ci/cluster-template-prow-custom-vnet.yaml | 8 ++++++++ .../ci/cluster-template-prow-dual-stack.yaml | 8 ++++++++ .../ci/cluster-template-prow-edgezone.yaml | 8 ++++++++ .../cluster-template-prow-flatcar-sysext.yaml | 8 ++++++++ .../ci/cluster-template-prow-flatcar.yaml | 8 ++++++++ .../test/ci/cluster-template-prow-ipv6.yaml | 8 ++++++++ ...template-prow-machine-pool-ci-version.yaml | 8 ++++++++ ...uster-template-prow-machine-pool-flex.yaml | 8 ++++++++ .../cluster-template-prow-machine-pool.yaml | 8 ++++++++ .../ci/cluster-template-prow-nvidia-gpu.yaml | 8 ++++++++ .../ci/cluster-template-prow-private.yaml | 8 ++++++++ .../test/ci/cluster-template-prow-spot.yaml | 8 ++++++++ templates/test/ci/cluster-template-prow.yaml | 16 +++++++++++++++ .../test/ci/patches/controller-manager.yaml | 8 ++++++++ .../patches/kubeadm-config-template-azl3.yaml | 8 ++++++++ ...uster-template-custom-builds-load-dra.yaml | 16 +++++++++++++++ .../cluster-template-custom-builds-load.yaml | 16 +++++++++++++++ .../dev/cluster-template-custom-builds.yaml | 16 +++++++++++++++ test/e2e/azure_test.go | 20 +++++++++---------- 27 files changed, 282 insertions(+), 10 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index 7fdfc44983c..6f5cfdf567c 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -231,6 +231,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index 73103749722..d3201145b1f 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -158,6 +158,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 3692b99c627..6efec5e0799 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -145,6 +145,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 156dadaccf6..fece6d6c9b5 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -263,6 +263,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 13ec248a995..4ec1d80771a 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -267,6 +267,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -523,6 +531,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index bca83e482d7..a18367f616d 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -274,6 +274,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -541,6 +549,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 653aa6c4c1d..77ba1f4e2eb 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -246,6 +246,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -500,6 +508,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 531e4e55a41..9dd5d621d3b 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -246,6 +246,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -500,6 +508,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 439e6ba7cbf..5463728e7cb 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -151,6 +151,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 01d11b58484..c88e49770bb 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -165,6 +165,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 9b87c5b6f97..766c3f15f45 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -147,6 +147,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index 9855570294e..b9a071041ac 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -384,6 +384,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 6758a6562f0..0576bc9e265 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -157,6 +157,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index f695662801c..cd566433000 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -172,6 +172,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index ee45edf7169..2cb88cecb9f 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -245,6 +245,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index feb9e6e7d82..99fcbe1d362 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -148,6 +148,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 06183fef377..bfd46973226 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -148,6 +148,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index ae1b9e2f9f0..e17938c7637 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -145,6 +145,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 759ac0f20f6..dce61f56a03 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -178,6 +178,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 3ed78f92b6a..f4cfce5036a 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -144,6 +144,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 8e8a176074a..482e6a11865 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -148,6 +148,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 @@ -296,6 +304,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save --- diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index 3cb4b77c613..e9ca7c10a0f 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -40,6 +40,14 @@ spec: iptables -A INPUT -p tcp --dport 2380 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index 0e8243aee18..58cbb2c2b90 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -41,6 +41,14 @@ spec: iptables -A INPUT -p tcp --dport 2380 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index 6fbecdd1014..90a5230fd26 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -261,6 +261,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -486,6 +494,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 2ea598c38dd..4a6ef14c401 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -244,6 +244,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -458,6 +466,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 279ece19992..cbadf9b1e95 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -238,6 +238,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -452,6 +460,14 @@ spec: iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT + # Allow kubelet API communication (port 10250) + iptables -A INPUT -p tcp --dport 10250 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT + + # Allow kube-apiserver communication (port 6443) + iptables -A INPUT -p tcp --dport 6443 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 4cda5f5775b..c60e035e848 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -290,7 +290,7 @@ var _ = Describe("Workload cluster creation", func() { withFlavor("azure-cni-v1"), withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(3), + withControlPlaneMachineCount(1), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ @@ -307,15 +307,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) - By("can expect VM extensions are present on the node", func() { - AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { - return AzureVMExtensionsSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } - }) - }) + // By("can expect VM extensions are present on the node", func() { + // AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + // return AzureVMExtensionsSpecInput{ + // BootstrapClusterProxy: bootstrapClusterProxy, + // Namespace: namespace, + // ClusterName: clusterName, + // } + // }) + // }) By("can validate failure domains", func() { AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { From ccd6d0655cc3c7de24e9b51d47b842a8a9a53a2a Mon Sep 17 00:00:00 2001 From: William Yao Date: Thu, 21 Aug 2025 11:43:59 -0700 Subject: [PATCH 09/19] iptables again --- ...late-prow-apiserver-ilb-custom-images.yaml | 10 ++++++++++ .../cluster-template-prow-apiserver-ilb.yaml | 10 ++++++++++ .../cluster-template-prow-azure-cni-v1.yaml | 10 ++++++++++ .../cluster-template-prow-ci-version-dra.yaml | 10 ++++++++++ ...r-template-prow-ci-version-dual-stack.yaml | 20 +++++++++++++++++++ ...cluster-template-prow-ci-version-ipv6.yaml | 20 +++++++++++++++++++ ...er-template-prow-ci-version-md-and-mp.yaml | 20 +++++++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 20 +++++++++++++++++++ .../ci/cluster-template-prow-custom-vnet.yaml | 10 ++++++++++ .../ci/cluster-template-prow-dual-stack.yaml | 10 ++++++++++ .../ci/cluster-template-prow-edgezone.yaml | 10 ++++++++++ .../cluster-template-prow-flatcar-sysext.yaml | 10 ++++++++++ .../ci/cluster-template-prow-flatcar.yaml | 10 ++++++++++ .../test/ci/cluster-template-prow-ipv6.yaml | 10 ++++++++++ ...template-prow-machine-pool-ci-version.yaml | 10 ++++++++++ ...uster-template-prow-machine-pool-flex.yaml | 10 ++++++++++ .../cluster-template-prow-machine-pool.yaml | 10 ++++++++++ .../ci/cluster-template-prow-nvidia-gpu.yaml | 10 ++++++++++ .../ci/cluster-template-prow-private.yaml | 10 ++++++++++ .../test/ci/cluster-template-prow-spot.yaml | 10 ++++++++++ templates/test/ci/cluster-template-prow.yaml | 20 +++++++++++++++++++ .../test/ci/patches/controller-manager.yaml | 10 ++++++++++ .../patches/kubeadm-config-template-azl3.yaml | 10 ++++++++++ ...uster-template-custom-builds-load-dra.yaml | 20 +++++++++++++++++++ .../cluster-template-custom-builds-load.yaml | 20 +++++++++++++++++++ .../dev/cluster-template-custom-builds.yaml | 20 +++++++++++++++++++ 26 files changed, 340 insertions(+) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index 6f5cfdf567c..2e45c6e4ce5 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -239,6 +239,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index d3201145b1f..559c07a92da 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -166,6 +166,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 6efec5e0799..dfa7f898db9 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -153,6 +153,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index fece6d6c9b5..424fa115387 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -271,6 +271,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 4ec1d80771a..887ef1fae8c 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -275,6 +275,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -539,6 +549,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index a18367f616d..42a437aff34 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -282,6 +282,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -557,6 +567,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index 77ba1f4e2eb..e7e04e3d0dd 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -254,6 +254,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -516,6 +526,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 9dd5d621d3b..7ae1fc69a4d 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -254,6 +254,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 @@ -516,6 +526,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 5463728e7cb..4a434775ee6 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -159,6 +159,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index c88e49770bb..f0c32abf10b 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -173,6 +173,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 766c3f15f45..2376b668bfe 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -155,6 +155,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index b9a071041ac..d83c5802bc0 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -392,6 +392,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 0576bc9e265..61162d41101 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -165,6 +165,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index cd566433000..2d31db5ed57 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -180,6 +180,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 2cb88cecb9f..68deb64ffaa 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -253,6 +253,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 5 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 99fcbe1d362..adf81c8139c 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -156,6 +156,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index bfd46973226..9f0232cd7a4 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -156,6 +156,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index e17938c7637..0ae11466302 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -153,6 +153,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index dce61f56a03..bf8851df59b 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -186,6 +186,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index f4cfce5036a..5df6b9d42ca 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -152,6 +152,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 482e6a11865..ec6efd9252a 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -156,6 +156,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save verbosity: 10 @@ -312,6 +322,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save --- diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index e9ca7c10a0f..ba4ad827f08 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -48,6 +48,16 @@ spec: # Allow kube-apiserver (port 6443) iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index 58cbb2c2b90..0540e750ec8 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -49,6 +49,16 @@ spec: # Allow kube-apiserver communication (port 6443) iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index 90a5230fd26..b7f9087bdb4 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -269,6 +269,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -502,6 +512,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 4a6ef14c401..25dfab1b4e3 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -252,6 +252,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -474,6 +484,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index cbadf9b1e95..f215f01428b 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -246,6 +246,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh @@ -468,6 +478,16 @@ spec: iptables -A INPUT -p tcp --dport 6443 -j ACCEPT iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT + # Allow nodeport range (30000-32767) + iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT + iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT + + # Allow all traffic between pod/service subnets + iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT + iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT + iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh From e8d99dd4f298923beed07f3fadc1ac7aa30c1c29 Mon Sep 17 00:00:00 2001 From: William Yao Date: Thu, 21 Aug 2025 15:01:27 -0700 Subject: [PATCH 10/19] follow docs exactly --- ...late-prow-apiserver-ilb-custom-images.yaml | 45 ++-------- .../cluster-template-prow-apiserver-ilb.yaml | 45 ++-------- .../cluster-template-prow-azure-cni-v1.yaml | 45 ++-------- .../cluster-template-prow-ci-version-dra.yaml | 45 ++-------- ...r-template-prow-ci-version-dual-stack.yaml | 90 +++---------------- ...cluster-template-prow-ci-version-ipv6.yaml | 90 +++---------------- ...er-template-prow-ci-version-md-and-mp.yaml | 90 +++---------------- .../ci/cluster-template-prow-ci-version.yaml | 90 +++---------------- .../ci/cluster-template-prow-custom-vnet.yaml | 45 ++-------- .../ci/cluster-template-prow-dual-stack.yaml | 45 ++-------- .../ci/cluster-template-prow-edgezone.yaml | 45 ++-------- .../cluster-template-prow-flatcar-sysext.yaml | 45 ++-------- .../ci/cluster-template-prow-flatcar.yaml | 45 ++-------- .../test/ci/cluster-template-prow-ipv6.yaml | 45 ++-------- ...template-prow-machine-pool-ci-version.yaml | 45 ++-------- ...uster-template-prow-machine-pool-flex.yaml | 45 ++-------- .../cluster-template-prow-machine-pool.yaml | 45 ++-------- .../ci/cluster-template-prow-nvidia-gpu.yaml | 45 ++-------- .../ci/cluster-template-prow-private.yaml | 45 ++-------- .../test/ci/cluster-template-prow-spot.yaml | 45 ++-------- templates/test/ci/cluster-template-prow.yaml | 90 +++---------------- .../test/ci/patches/controller-manager.yaml | 45 ++-------- .../patches/kubeadm-config-template-azl3.yaml | 45 ++-------- ...uster-template-custom-builds-load-dra.yaml | 90 +++---------------- .../cluster-template-custom-builds-load.yaml | 90 +++---------------- .../dev/cluster-template-custom-builds.yaml | 90 +++---------------- 26 files changed, 170 insertions(+), 1360 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index 2e45c6e4ce5..338e4aa6e2d 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -200,54 +200,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index 559c07a92da..c4b83a785ad 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -127,54 +127,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index dfa7f898db9..2d7abdce347 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -114,54 +114,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 424fa115387..f4c240a5b49 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -232,54 +232,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 887ef1fae8c..b07edfdcc14 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -236,54 +236,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -510,54 +475,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 42a437aff34..00971951b17 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -243,54 +243,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -528,54 +493,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index e7e04e3d0dd..af04fe0b78f 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -215,54 +215,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -487,54 +452,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 7ae1fc69a4d..d90b5a14c15 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -215,54 +215,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -487,54 +452,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 4a434775ee6..5616b77ca92 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -120,54 +120,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index f0c32abf10b..727cd78f06b 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -134,54 +134,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 2376b668bfe..89922f45986 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -116,54 +116,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index d83c5802bc0..e2e2f98113b 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -353,54 +353,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 61162d41101..6f94b2ba800 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -126,54 +126,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 2d31db5ed57..3075e7ffe00 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -141,54 +141,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 68deb64ffaa..eebf6533eda 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -214,54 +214,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index adf81c8139c..0a457f14286 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -117,54 +117,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 9f0232cd7a4..7c862e8b2b4 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -117,54 +117,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 0ae11466302..be5786c6e7f 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -114,54 +114,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index bf8851df59b..ceb5a0216fe 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -147,54 +147,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 5df6b9d42ca..1aadc282c07 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -113,54 +113,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index ec6efd9252a..2e388e27af8 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -117,54 +117,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -283,54 +248,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index ba4ad827f08..1c38fdc8dd7 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -10,54 +10,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml index 0540e750ec8..d76a784f62b 100644 --- a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/patches/kubeadm-config-template-azl3.yaml @@ -11,54 +11,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index b7f9087bdb4..a7d640960e8 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -230,54 +230,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -473,54 +438,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 25dfab1b4e3..ee990090f8f 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -213,54 +213,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -445,54 +410,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index f215f01428b..761930d86f2 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -207,54 +207,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save @@ -439,54 +404,19 @@ spec: tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - # Completely reset iptables to permissive state for etcd connectivity - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -t raw -F 2>/dev/null || true - iptables -t raw -X 2>/dev/null || true + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT iptables -P FORWARD ACCEPT iptables -P OUTPUT ACCEPT - ip6tables -F - ip6tables -X - ip6tables -t nat -F - ip6tables -t nat -X - ip6tables -t mangle -F - ip6tables -t mangle -X - ip6tables -t raw -F 2>/dev/null || true - ip6tables -t raw -X 2>/dev/null || true ip6tables -P INPUT ACCEPT ip6tables -P FORWARD ACCEPT ip6tables -P OUTPUT ACCEPT - # Allow all etcd communication explicitly (for worker nodes that might become control planes) - iptables -A INPUT -p tcp --dport 2379 -j ACCEPT - iptables -A INPUT -p tcp --dport 2380 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2379 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 2380 -j ACCEPT - - # Allow kubelet API communication (port 10250) - iptables -A INPUT -p tcp --dport 10250 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 10250 -j ACCEPT - - # Allow kube-apiserver communication (port 6443) - iptables -A INPUT -p tcp --dport 6443 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 6443 -j ACCEPT - - # Allow nodeport range (30000-32767) - iptables -A INPUT -p tcp --dport 30000:32767 -j ACCEPT - iptables -A OUTPUT -p tcp --dport 30000:32767 -j ACCEPT - - # Allow all traffic between pod/service subnets - iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT - iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT - iptables -A FORWARD -d 10.0.0.0/8 -j ACCEPT + # Flush any rules which would filter packets + iptables -F + ip6tables -F iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save From e3227a2fed6afe42208a91b9317973584810f680 Mon Sep 17 00:00:00 2001 From: William Yao Date: Thu, 21 Aug 2025 18:09:42 -0700 Subject: [PATCH 11/19] Try making new template --- azure/defaults.go | 2 +- .../cloud-provider-azure-ci.yaml | 1 - .../cloud-provider-azure.yaml | 1 - templates/cluster-template-aad.yaml | 1 - templates/cluster-template-apiserver-ilb.yaml | 1 - templates/cluster-template-azure-bastion.yaml | 2 - templates/cluster-template-azure-cni-v1.yaml | 2 - templates/cluster-template-dual-stack.yaml | 1 - templates/cluster-template-edgezone.yaml | 2 - templates/cluster-template-ephemeral.yaml | 2 - .../cluster-template-flatcar-sysext.yaml | 1 - templates/cluster-template-flatcar.yaml | 1 - templates/cluster-template-ipv6.yaml | 1 - .../cluster-template-machinepool-windows.yaml | 1 - templates/cluster-template-machinepool.yaml | 1 - templates/cluster-template-nvidia-gpu.yaml | 1 - templates/cluster-template-private.yaml | 2 - ...luster-template-windows-apiserver-ilb.yaml | 1 - templates/cluster-template-windows.yaml | 2 - templates/cluster-template.yaml | 2 - templates/flavors/base/cluster-template.yaml | 1 - .../flavors/default/machine-deployment.yaml | 1 - ...late-prow-apiserver-ilb-custom-images.yaml | 24 - .../cluster-template-prow-apiserver-ilb.yaml | 26 +- .../test/ci/cluster-template-prow-azl3.yaml | 415 ++++++++++++++++++ .../cluster-template-prow-azure-cni-v1.yaml | 27 +- .../cluster-template-prow-ci-version-dra.yaml | 26 +- ...r-template-prow-ci-version-dual-stack.yaml | 48 +- ...cluster-template-prow-ci-version-ipv6.yaml | 48 +- ...er-template-prow-ci-version-md-and-mp.yaml | 48 +- .../ci/cluster-template-prow-ci-version.yaml | 48 +- ...om-builds-apiserver-ilb-custom-images.yaml | 0 .../ci/cluster-template-prow-custom-vnet.yaml | 27 +- .../ci/cluster-template-prow-dual-stack.yaml | 26 +- .../ci/cluster-template-prow-edgezone.yaml | 27 +- .../cluster-template-prow-flatcar-sysext.yaml | 24 +- .../ci/cluster-template-prow-flatcar.yaml | 24 +- .../test/ci/cluster-template-prow-ipv6.yaml | 26 +- ...template-prow-machine-pool-ci-version.yaml | 26 +- ...uster-template-prow-machine-pool-flex.yaml | 26 +- .../cluster-template-prow-machine-pool.yaml | 26 +- .../ci/cluster-template-prow-nvidia-gpu.yaml | 26 +- .../ci/cluster-template-prow-private.yaml | 27 +- .../test/ci/cluster-template-prow-spot.yaml | 27 +- .../ci/cluster-template-prow-topology.yaml | 2 - templates/test/ci/cluster-template-prow.yaml | 50 +-- .../test/ci/patches/controller-manager.yaml | 22 - .../test/ci/prow-azl3/kustomization.yaml | 26 ++ .../azuremachinetemplate-azl3-image.yaml | 25 ++ .../cloud-provider-azure-cacertdir.yaml | 12 + .../cloud-provider-azure-ci-cacertdir.yaml | 23 + .../prow-azl3/patches/controller-manager.yaml | 28 ++ .../disable-vm-bootstrap-extension.yaml | 17 + .../patches/kubeadm-config-template-azl3.yaml | 0 templates/test/ci/prow/kustomization.yaml | 1 - .../cluster-template-custom-builds-dra.yaml | 3 - ...uster-template-custom-builds-load-dra.yaml | 46 -- .../cluster-template-custom-builds-load.yaml | 46 -- ...e-custom-builds-machine-pool-load-dra.yaml | 3 - ...plate-custom-builds-machine-pool-load.yaml | 3 - ...r-template-custom-builds-machine-pool.yaml | 3 - .../dev/cluster-template-custom-builds.yaml | 46 -- test/e2e/azure_test.go | 20 +- test/e2e/config/azure-dev.yaml | 2 + .../cluster-template-kcp-remediation.yaml | 2 - .../cluster-template-kcp-scale-in.yaml | 2 - ...ter-template-machine-and-machine-pool.yaml | 2 - .../cluster-template-machine-pool.yaml | 2 - .../cluster-template-md-remediation.yaml | 2 - .../v1beta1/cluster-template-node-drain.yaml | 2 - .../v1beta1/cluster-template-upgrades.yaml | 2 - .../v1beta1/cluster-template.yaml | 2 - 72 files changed, 590 insertions(+), 856 deletions(-) create mode 100644 templates/test/ci/cluster-template-prow-azl3.yaml delete mode 100644 templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml create mode 100644 templates/test/ci/prow-azl3/kustomization.yaml create mode 100644 templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml create mode 100644 templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml create mode 100644 templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml create mode 100644 templates/test/ci/prow-azl3/patches/controller-manager.yaml create mode 100644 templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml rename templates/test/ci/{ => prow-azl3}/patches/kubeadm-config-template-azl3.yaml (100%) diff --git a/azure/defaults.go b/azure/defaults.go index f5b28b95a41..02e5508fa5c 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -50,7 +50,7 @@ const ( // DefaultPublicGalleryName is the default Azure compute gallery. DefaultPublicGalleryName = "ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019" // DefaultLinuxGalleryImageName is the default Linux community gallery image definition. - DefaultLinuxGalleryImageName = "capi-azurelinux-3" + DefaultLinuxGalleryImageName = "capi-ubun2-2404" // DefaultWindowsGalleryImageName is the default Windows community gallery image definition. DefaultWindowsGalleryImageName = "capi-win-2019-containerd" ) diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml index f04a8c6558d..04cfc66b24a 100644 --- a/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml +++ b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml @@ -13,7 +13,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml index e729586df93..7838783312e 100644 --- a/templates/addons/cluster-api-helm/cloud-provider-azure.yaml +++ b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml @@ -13,6 +13,5 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index ae14e6047a1..24451adadfb 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -126,7 +126,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-apiserver-ilb.yaml b/templates/cluster-template-apiserver-ilb.yaml index af1a74a41d4..6339dc4a392 100644 --- a/templates/cluster-template-apiserver-ilb.yaml +++ b/templates/cluster-template-apiserver-ilb.yaml @@ -135,7 +135,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 8d11b47abd0..f00edd6f71d 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -123,7 +123,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -165,7 +164,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index d6a66f70080..55e31104921 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -123,7 +123,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned networkInterfaces: - privateIPConfigs: 110 @@ -168,7 +167,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true networkInterfaces: - privateIPConfigs: 110 subnetName: node-subnet diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index 0381af0aa2c..c5aac3e9941 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -144,7 +144,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index a102377c944..f06ebfd315b 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -124,7 +124,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -166,7 +165,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index e05f43ed523..7305b98a83e 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -121,7 +121,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: cachingType: ReadOnly @@ -166,7 +165,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: cachingType: ReadOnly diffDiskSettings: diff --git a/templates/cluster-template-flatcar-sysext.yaml b/templates/cluster-template-flatcar-sysext.yaml index e2a2d58eecc..5bcebbf76a5 100644 --- a/templates/cluster-template-flatcar-sysext.yaml +++ b/templates/cluster-template-flatcar-sysext.yaml @@ -290,7 +290,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 86b87c7d641..2efd2a4d43e 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -136,7 +136,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index 09bc7230a1a..0be25635b0e 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -149,7 +149,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index ae64fea8f05..d4582954b92 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -125,7 +125,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index e7255ef6161..a7caeff7c25 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -121,7 +121,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index a3a73798141..3cc10d76386 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -121,7 +121,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 947c1d400ae..7dbd441dbd6 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -135,7 +135,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -177,7 +176,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template-windows-apiserver-ilb.yaml b/templates/cluster-template-windows-apiserver-ilb.yaml index 08c5ee87771..30302ec00d4 100644 --- a/templates/cluster-template-windows-apiserver-ilb.yaml +++ b/templates/cluster-template-windows-apiserver-ilb.yaml @@ -139,7 +139,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 8241951320a..66c184ede74 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -125,7 +125,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -167,7 +166,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a0ab1ce73a6..b970c267af6 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -121,7 +121,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -163,7 +162,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index b83012e3803..055a57e247d 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -109,7 +109,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} osDisk: osType: "Linux" diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index 216d6511c01..9aaf668473f 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -29,7 +29,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true vmSize: ${AZURE_NODE_MACHINE_TYPE} osDisk: osType: "Linux" diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml index 338e4aa6e2d..9af9dfcce2e 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml @@ -195,27 +195,6 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -238,7 +217,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -489,7 +467,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -509,7 +486,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml index c4b83a785ad..b0db2aef17b 100644 --- a/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml +++ b/templates/test/ci/cluster-template-prow-apiserver-ilb.yaml @@ -121,28 +121,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -164,7 +143,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -380,7 +358,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -400,7 +377,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-azl3.yaml b/templates/test/ci/cluster-template-prow-azl3.yaml new file mode 100644 index 00000000000..e8cb04eafe1 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-azl3.yaml @@ -0,0 +1,415 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: {} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + verbosity: 10 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + disableVMBootstrapExtension: true + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + disableVMBootstrapExtension: true + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: mcr.microsoft.com/oss + # Image and registry configuration for the tigera/operator pod. + tigeraOperator: + image: tigera/operator + registry: mcr.microsoft.com/oss + calicoctl: + image: mcr.microsoft.com/oss/calico/ctl + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 2d7abdce347..d512f1c6ba3 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -108,28 +108,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -151,7 +130,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned networkInterfaces: - privateIPConfigs: 110 @@ -196,7 +174,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true networkInterfaces: - privateIPConfigs: 110 subnetName: node-subnet @@ -298,7 +275,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -318,7 +294,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index f4c240a5b49..ca736d04faf 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -227,27 +227,8 @@ spec: postKubeadmCommands: [] preKubeadmCommands: - bash -c /tmp/containerd-config.sh - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -269,7 +250,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -825,7 +805,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -845,7 +824,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index b07edfdcc14..04e4a9f445d 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -231,27 +231,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -275,7 +256,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -328,7 +308,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -470,27 +449,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -607,7 +565,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -627,7 +584,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 00971951b17..cfa5ecdd107 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -238,27 +238,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -282,7 +263,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -335,7 +315,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned image: @@ -488,27 +467,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -625,7 +583,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -645,7 +602,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index af04fe0b78f..0a873707764 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -210,27 +210,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -254,7 +235,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -306,7 +286,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -447,27 +426,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -857,7 +815,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -877,7 +834,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index d90b5a14c15..a6371798ed7 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -210,27 +210,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -254,7 +235,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -306,7 +286,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -447,27 +426,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 @@ -857,7 +815,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -877,7 +834,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml b/templates/test/ci/cluster-template-prow-custom-builds-apiserver-ilb-custom-images.yaml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 5616b77ca92..a87c9da641b 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -114,28 +114,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -157,7 +136,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -201,7 +179,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -371,7 +348,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -391,7 +367,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 727cd78f06b..c6da620e0dd 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -128,28 +128,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -171,7 +150,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: @@ -460,7 +438,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -480,7 +457,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 89922f45986..7c46b6bdc86 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -110,28 +110,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -155,7 +134,6 @@ spec: managedDisk: storageAccountType: StandardSSD_LRS nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -198,7 +176,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -354,7 +331,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -374,7 +350,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml index e2e2f98113b..d4c2258f876 100644 --- a/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml @@ -348,27 +348,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml verbosity: 10 machineTemplate: infrastructureRef: @@ -429,7 +410,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 6f94b2ba800..8a479d69bb5 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -121,27 +121,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' + | jq -r .compute.name)/g" /etc/kubeadm.yml verbosity: 10 machineTemplate: infrastructureRef: @@ -163,7 +144,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 3075e7ffe00..b8e8fe10ecb 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -135,28 +135,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -178,7 +157,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true enableIPForwarding: true identity: UserAssigned osDisk: @@ -480,7 +458,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -500,7 +477,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index eebf6533eda..7a9648382bc 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -209,27 +209,8 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -251,7 +232,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -785,7 +765,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -805,7 +784,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 0a457f14286..d135e419b2d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -111,28 +111,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -154,7 +133,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -520,7 +498,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -540,7 +517,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 7c862e8b2b4..4b1ad679642 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -111,28 +111,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -154,7 +133,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -514,7 +492,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -534,7 +511,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index be5786c6e7f..8ea79155209 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -108,28 +108,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -151,7 +130,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -344,7 +322,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -364,7 +341,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index ceb5a0216fe..20a7a0d8e1c 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -142,27 +142,8 @@ spec: ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io apiserver' >> /etc/hosts; fi preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - if [ -f /tmp/kubeadm.yaml ] || [ -f /run/kubeadm/kubeadm.yaml ]; then echo '127.0.0.1 apiserver.${CLUSTER_NAME}.capz.io + apiserver' >> /etc/hosts; fi verbosity: 10 machineTemplate: infrastructureRef: @@ -184,7 +165,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -226,7 +206,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -394,7 +373,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -414,7 +392,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 1aadc282c07..dcd6b559e2c 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -107,28 +107,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -150,7 +129,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -194,7 +172,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -367,7 +344,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -387,7 +363,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index 2ebd68f9095..b1ffd57246e 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -189,7 +189,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -209,7 +208,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 2e388e27af8..6be8c8d38f6 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -111,28 +111,7 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] verbosity: 10 machineTemplate: infrastructureRef: @@ -156,7 +135,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -202,7 +180,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true identity: UserAssigned osDisk: diskSizeGB: 128 @@ -242,28 +219,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + preKubeadmCommands: [] --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -591,7 +547,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -611,7 +566,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/ci/patches/controller-manager.yaml b/templates/test/ci/patches/controller-manager.yaml index 1c38fdc8dd7..007509ef572 100644 --- a/templates/test/ci/patches/controller-manager.yaml +++ b/templates/test/ci/patches/controller-manager.yaml @@ -4,28 +4,6 @@ metadata: name: "${CLUSTER_NAME}-control-plane" spec: kubeadmConfigSpec: - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save clusterConfiguration: controllerManager: extraArgs: diff --git a/templates/test/ci/prow-azl3/kustomization.yaml b/templates/test/ci/prow-azl3/kustomization.yaml new file mode 100644 index 00000000000..9ec870f399f --- /dev/null +++ b/templates/test/ci/prow-azl3/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../flavors/default +- ../../../addons/cluster-api-helm/calico.yaml +- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml +patches: +- path: ../patches/tags.yaml +- path: ../patches/mhc.yaml +- path: ../patches/controller-manager.yaml +- path: ../patches/uami-md-0.yaml +- path: ../patches/uami-control-plane.yaml +- path: ../patches/cluster-label-calico.yaml +- path: ../patches/cluster-label-cloud-provider-azure.yaml +- path: patches/controller-manager.yaml +- path: patches/kubeadm-config-template-azl3.yaml +- path: patches/azuremachinetemplate-azl3-image.yaml +- path: patches/cloud-provider-azure-cacertdir.yaml +- path: patches/cloud-provider-azure-ci-cacertdir.yaml +- path: patches/disable-vm-bootstrap-extension.yaml + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml new file mode 100644 index 00000000000..143e4be2dfc --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml @@ -0,0 +1,25 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} diff --git a/templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml b/templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml new file mode 100644 index 00000000000..1a19310be5e --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml @@ -0,0 +1,12 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 diff --git a/templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml b/templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml new file mode 100644 index 00000000000..cf26adf88c4 --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml @@ -0,0 +1,23 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" diff --git a/templates/test/ci/prow-azl3/patches/controller-manager.yaml b/templates/test/ci/prow-azl3/patches/controller-manager.yaml new file mode 100644 index 00000000000..a064ec5792e --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/controller-manager.yaml @@ -0,0 +1,28 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - | + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save diff --git a/templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml b/templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml new file mode 100644 index 00000000000..5d06409a84d --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml @@ -0,0 +1,17 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + disableVMBootstrapExtension: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + disableVMBootstrapExtension: true diff --git a/templates/test/ci/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml similarity index 100% rename from templates/test/ci/patches/kubeadm-config-template-azl3.yaml rename to templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 5ca27992f0d..81f7b28fb41 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -50,7 +50,6 @@ patches: - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml - path: ../patches/uami-md-0.yaml -- path: ../patches/kubeadm-config-template-azl3.yaml configMapGenerator: - files: - windows-cni=../../../addons/windows/calico/calico.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-dra.yaml b/templates/test/dev/cluster-template-custom-builds-dra.yaml index ac9fe980730..e703f6574cb 100644 --- a/templates/test/dev/cluster-template-custom-builds-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-dra.yaml @@ -253,7 +253,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -760,7 +759,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -780,7 +778,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index a7d640960e8..a255a7bc590 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -225,27 +225,6 @@ spec: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - bash -c /tmp/containerd-config.sh - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -271,7 +250,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -323,7 +301,6 @@ spec: spec: additionalTags: monitoring: dra - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -433,27 +410,6 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/containerd-config.sh - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -869,7 +825,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -889,7 +844,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index ee990090f8f..dfb0194c462 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -208,27 +208,6 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -254,7 +233,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -306,7 +284,6 @@ spec: spec: additionalTags: monitoring: load - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -405,27 +382,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -831,7 +787,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -851,7 +806,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml index 590a86a6153..ada9bba7833 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml @@ -261,7 +261,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -770,7 +769,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -790,7 +788,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml index ec5cd52581b..6d6dec5a232 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml @@ -243,7 +243,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -730,7 +729,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -750,7 +748,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 13a9ea82295..54dfc708b3f 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -235,7 +235,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -720,7 +719,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -740,7 +738,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 761930d86f2..b1ad2a278f3 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -202,27 +202,6 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 @@ -248,7 +227,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -300,7 +278,6 @@ spec: spec: additionalTags: monitoring: virtualmachine - disableVMBootstrapExtension: true identity: UserAssigned image: marketplace: @@ -399,27 +376,6 @@ spec: image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- @@ -825,7 +781,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -845,7 +800,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index c60e035e848..f2683207379 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -198,12 +198,14 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating a highly available cluster [REQUIRED]", func() { It("With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes", func() { + Expect(os.Setenv("KUBERNETES_VERSION", "v1.33.2")).To(Succeed()) clusterName = getClusterName(clusterNamePrefix, "ha") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, withNamespace(namespace.Name), withClusterName(clusterName), + withFlavor("azl3"), withControlPlaneMachineCount(1), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), @@ -221,15 +223,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) - By("Verifying expected VM extensions are present on the node", func() { - AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { - return AzureVMExtensionsSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } - }) - }) + // By("Verifying expected VM extensions are present on the node", func() { + // AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + // return AzureVMExtensionsSpecInput{ + // BootstrapClusterProxy: bootstrapClusterProxy, + // Namespace: namespace, + // ClusterName: clusterName, + // } + // }) + // }) By("Verifying security rules are deleted on azure side", func() { AzureSecurityGroupsSpec(ctx, func() AzureSecurityGroupsSpecInput { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index c868a09592e..58a4c59d815 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -184,6 +184,8 @@ providers: targetName: "cluster-template-apiserver-ilb.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-apiserver-ilb-custom-images.yaml" targetName: "cluster-template-apiserver-ilb-custom-images.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azl3.yaml" + targetName: "cluster-template-azl3.yaml" replacements: - old: "--v=0" new: "--v=2" diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml index 53056b07df1..778e4527583 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml index 73e18e8de23..087423e72ee 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml index ac2ee9ff492..26f72ca9b12 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml index 950986fafca..71b94a0dee5 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml index 41a5dbf0f6e..59d5e557127 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml index 6fea9c4ff75..629ed5f1960 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml index faa913963a0..01ceecbf9e8 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml index 2a0d1c8c65e..89c45b8c1b1 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml @@ -103,7 +103,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} logVerbosity: 4 --- @@ -122,7 +121,6 @@ spec: infra: clusterName: {{ .Cluster.metadata.name }} cloudControllerManager: - caCertDir: "/etc/pki/tls/certs" cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} From 16ab1d9b60b2429a11ae2ae296cec82aa94382e9 Mon Sep 17 00:00:00 2001 From: William Yao Date: Fri, 22 Aug 2025 11:58:05 -0700 Subject: [PATCH 12/19] Try 3 cp --- test/e2e/azure_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index f2683207379..2b16516ad9f 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -199,6 +199,7 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating a highly available cluster [REQUIRED]", func() { It("With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes", func() { Expect(os.Setenv("KUBERNETES_VERSION", "v1.33.2")).To(Succeed()) + Expect(os.Setenv("SKIP_CLEANUP", "true")).To(Succeed()) clusterName = getClusterName(clusterNamePrefix, "ha") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( @@ -206,7 +207,7 @@ var _ = Describe("Workload cluster creation", func() { withNamespace(namespace.Name), withClusterName(clusterName), withFlavor("azl3"), - withControlPlaneMachineCount(1), + withControlPlaneMachineCount(3), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ From ab0767857004b7ebb97cc151b09234cd9fe714c6 Mon Sep 17 00:00:00 2001 From: William Yao Date: Sun, 24 Aug 2025 13:54:08 -0700 Subject: [PATCH 13/19] Update log collector --- test/e2e/azure_logcollector.go | 54 ++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index e0507cae1cd..1ac239b019f 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -443,15 +443,63 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ... ), execToPathFn( "cloud-init.log", - "cat", "/var/log/cloud-init.log", + "sudo", "sh", "-c", "if [ -f /var/log/cloud-init.log ]; then sudo cat /var/log/cloud-init.log; else echo 'cloud-init.log not found'; fi", ), execToPathFn( "cloud-init-output.log", - "cat", "/var/log/cloud-init-output.log", + "sudo", "sh", "-c", "echo 'Waiting for cloud-init to complete before collecting output log...' && cloud-init status --wait && echo 'Cloud-init completed, collecting output log...' && if [ -f /var/log/cloud-init-output.log ]; then echo 'Found cloud-init-output.log, reading contents:' && sudo cat /var/log/cloud-init-output.log; else echo 'cloud-init-output.log not found after cloud-init completion'; fi", + ), + execToPathFn( + "cloud-init-journal.log", + "sudo", "journalctl", "--no-pager", "--output=short-precise", "-u", "cloud-init", "-u", "cloud-config", "-u", "cloud-final", + ), + execToPathFn( + "cloud-init-status.txt", + "sudo", "cloud-init", "status", "--long", + ), + execToPathFn( + "cloud-init-all-logs.txt", + "sudo", "sh", "-c", "echo '=== cloud-init logs from journal ===' && journalctl --no-pager -u cloud-init-local -u cloud-init -u cloud-config -u cloud-final --output=short-precise && echo && echo '=== cloud-init result.json ===' && cat /run/cloud-init/result.json 2>/dev/null || echo 'result.json not found' && echo && echo '=== cloud-init instance-data.json ===' && cat /run/cloud-init/instance-data.json 2>/dev/null || echo 'instance-data.json not found'", + ), + execToPathFn( + "cloud-init-file-details.txt", + "sudo", "sh", "-c", "echo '=== Cloud-init file existence and permissions ===' && echo 'Timestamp: '$(date) && echo 'Cloud-init status:' && cloud-init status && echo && echo 'Files in /var/log/ matching cloud-init*:' && ls -la /var/log/cloud-init* 2>/dev/null || echo 'No cloud-init files found in /var/log/' && echo && echo 'Files in /run/cloud-init/:' && ls -la /run/cloud-init/ 2>/dev/null || echo '/run/cloud-init/ not found' && echo && echo 'Checking for sentinel file:' && ls -la /run/cluster-api/ 2>/dev/null || echo '/run/cluster-api/ not found' && echo && echo 'SELinux context (if applicable):' && ls -laZ /var/log/cloud-init* 2>/dev/null || echo 'No SELinux or cloud-init files'", + ), + execToPathFn( + "cloud-init-output-comprehensive.log", + "sudo", "sh", "-c", "echo '=== Comprehensive cloud-init output collection ===' && echo 'Method 1: Direct sudo cat:' && sudo cat /var/log/cloud-init-output.log", + ), + execToPathFn( + "cloud-init-output-methods.log", + "sudo", "sh", "-c", "echo 'Method 2: sudo tail:' && sudo tail -n +1 /var/log/cloud-init-output.log && echo && echo 'Method 3: sudo dd:' && sudo dd if=/var/log/cloud-init-output.log 2>/dev/null && echo && echo 'Method 4: File readability test:' && sudo test -r /var/log/cloud-init-output.log && echo 'File readable with sudo' || echo 'File not readable with sudo'", + ), + execToPathFn( + "cloud-init-userdata.log", + "sudo", "cloud-init", "query", "userdata", ), execToPathFn( "sentinel-file-dir.txt", - "ls", "/run/cluster-api/", + "ls", "-la", "/run/cluster-api/", + ), + execToPathFn( + "var-log-dir.txt", + "ls", "-la", "/var/log/", + ), + execToPathFn( + "system-info.txt", + "sudo", "sh", "-c", "echo '=== OS Release ===' && cat /etc/os-release && echo && echo '=== Uptime ===' && uptime && echo && echo '=== Free memory ===' && free -h", + ), + execToPathFn( + "dmesg.log", + "sudo", "dmesg", + ), + execToPathFn( + "systemd-analyze.txt", + "sudo", "systemd-analyze", "blame", + ), + execToPathFn( + "cloud-init-run-dir.txt", + "sudo", "ls", "-la", "/run/cloud-init/", ), execToPathFn( "cni.log", From 84c9010ffa71043048d4570bffe7190a2baa7e1a Mon Sep 17 00:00:00 2001 From: William Yao Date: Mon, 25 Aug 2025 10:12:46 -0700 Subject: [PATCH 14/19] Re-enable bootstrap extension --- templates/test/ci/cluster-template-prow-azl3.yaml | 2 -- templates/test/ci/prow-azl3/kustomization.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-azl3.yaml b/templates/test/ci/cluster-template-prow-azl3.yaml index e8cb04eafe1..098b8315be2 100644 --- a/templates/test/ci/cluster-template-prow-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-azl3.yaml @@ -150,7 +150,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: @@ -199,7 +198,6 @@ metadata: spec: template: spec: - disableVMBootstrapExtension: true identity: UserAssigned image: computeGallery: diff --git a/templates/test/ci/prow-azl3/kustomization.yaml b/templates/test/ci/prow-azl3/kustomization.yaml index 9ec870f399f..15a918e39b2 100644 --- a/templates/test/ci/prow-azl3/kustomization.yaml +++ b/templates/test/ci/prow-azl3/kustomization.yaml @@ -20,7 +20,6 @@ patches: - path: patches/azuremachinetemplate-azl3-image.yaml - path: patches/cloud-provider-azure-cacertdir.yaml - path: patches/cloud-provider-azure-ci-cacertdir.yaml -- path: patches/disable-vm-bootstrap-extension.yaml sortOptions: order: fifo From a9a1dceae3b148fc2e41d39d18416e0deec6d9e3 Mon Sep 17 00:00:00 2001 From: William Yao Date: Mon, 25 Aug 2025 11:48:10 -0700 Subject: [PATCH 15/19] Additional ssh connectivity test --- test/e2e/azure_logcollector.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 1ac239b019f..32efdf03803 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -52,6 +52,8 @@ type AzureLogCollector struct{} const ( collectLogInterval = 3 * time.Second collectLogTimeout = 1 * time.Minute + // extendedCollectLogTimeout is used for scenarios where nodes might take longer to be ready (e.g., additional control plane nodes) + extendedCollectLogTimeout = 3 * time.Minute ) var _ framework.ClusterLogCollector = &AzureLogCollector{} @@ -334,13 +336,20 @@ func collectLogsFromNode(cluster *clusterv1.Cluster, hostname string, isWindows execToPathFn := func(outputFileName, command string, args ...string) func() error { return func() error { - return retryWithTimeout(collectLogInterval, collectLogTimeout, func() error { + // Use extended timeout for better resilience, especially for additional control plane nodes + return retryWithTimeout(collectLogInterval, extendedCollectLogTimeout, func() error { f, err := fileOnHost(filepath.Join(outputPath, outputFileName)) if err != nil { + Logf("Failed to create output file %s for node %s: %v", outputFileName, hostname, err) return err } defer f.Close() - return execOnHost(controlPlaneEndpoint, hostname, sshPort, collectLogTimeout, f, command, args...) + + err = execOnHost(controlPlaneEndpoint, hostname, sshPort, extendedCollectLogTimeout, f, command, args...) + if err != nil { + Logf("Failed to execute command '%s %v' on node %s: %v", command, args, hostname, err) + } + return err }) } } @@ -417,6 +426,10 @@ func getAzureMachinePool(ctx context.Context, managementClusterClient client.Cli func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { return []func() error{ + execToPathFn( + "ssh-connectivity-test.txt", + "echo", "SSH connectivity test successful - $(date) - hostname: $(hostname) - uptime: $(uptime)", + ), execToPathFn( "journal.log", "sudo", "journalctl", "--no-pager", "--output=short-precise", @@ -511,6 +524,10 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ... "kube-apiserver.log", crictlPodLogsCmd("kube-apiserver"), ), + execToPathFn( + "log-collection-summary.txt", + "echo", "Log collection completed at $(date) for node $(hostname). SSH connectivity was successful if you can see this message.", + ), } } From 4ac224a13f32f78178a40538d6d10a624b6cd3a6 Mon Sep 17 00:00:00 2001 From: William Yao Date: Mon, 25 Aug 2025 13:55:48 -0700 Subject: [PATCH 16/19] Add conformance template and also test vm extension again --- .../test/ci/cluster-template-prow-azl3.yaml | 104 +- ...cluster-template-prow-ci-version-azl3.yaml | 1494 +++++++++++++++++ .../test/ci/prow-azl3/kustomization.yaml | 11 + .../prow-azl3/patches/controller-manager.yaml | 27 +- .../patches/kubeadm-config-template-azl3.yaml | 60 +- .../patches/remove-marketplace-image.yaml | 19 + .../prow-ci-version-azl3/kustomization.yaml | 26 + test/e2e/azure_logcollector.go | 21 +- test/e2e/azure_test.go | 38 +- test/e2e/config/azure-dev.yaml | 2 + test/e2e/conformance_test.go | 3 + 11 files changed, 1690 insertions(+), 115 deletions(-) create mode 100644 templates/test/ci/cluster-template-prow-ci-version-azl3.yaml create mode 100644 templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml create mode 100644 templates/test/ci/prow-ci-version-azl3/kustomization.yaml diff --git a/templates/test/ci/cluster-template-prow-azl3.yaml b/templates/test/ci/cluster-template-prow-azl3.yaml index 098b8315be2..9eac07241e7 100644 --- a/templates/test/ci/cluster-template-prow-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-azl3.yaml @@ -86,6 +86,36 @@ spec: overwrite: false tableType: gpt files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + owner: root:root + path: /tmp/azl3-setup.sh + permissions: "0744" - contentFrom: secret: key: control-plane-azure.json @@ -108,27 +138,7 @@ spec: - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/azl3-setup.sh verbosity: 10 machineTemplate: infrastructureRef: @@ -221,6 +231,36 @@ spec: template: spec: files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + owner: root:root + path: /tmp/azl3-setup.sh + permissions: "0744" - contentFrom: secret: key: worker-node-azure.json @@ -234,27 +274,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT - - # Flush any rules which would filter packets - iptables -F - ip6tables -F - - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + - bash -c /tmp/azl3-setup.sh --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml new file mode 100644 index 00000000000..71769fe589f --- /dev/null +++ b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml @@ -0,0 +1,1494 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + owner: root:root + path: /tmp/azl3-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/azl3-setup.sh + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + identity: UserAssigned + image: + computeGallery: + gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 + name: capi-azurelinux-3 + version: ${KUBERNETES_VERSION#v} + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + owner: root:root + path: /tmp/azl3-setup.sh + permissions: "0744" + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/azl3-setup.sh + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + identity: UserAssigned + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-ci-binaries.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 + - powershell C:/replace-ci-binaries.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: mcr.microsoft.com/oss + # Image and registry configuration for the tigera/operator pod. + tigeraOperator: + image: tigera/operator + registry: mcr.microsoft.com/oss + calicoctl: + image: mcr.microsoft.com/oss/calico/ctl + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + caCertDir: "/etc/pki/tls/certs" + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/prow-azl3/kustomization.yaml b/templates/test/ci/prow-azl3/kustomization.yaml index 15a918e39b2..98e08aeb1c2 100644 --- a/templates/test/ci/prow-azl3/kustomization.yaml +++ b/templates/test/ci/prow-azl3/kustomization.yaml @@ -16,7 +16,18 @@ patches: - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml - path: patches/controller-manager.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 - path: patches/kubeadm-config-template-azl3.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 - path: patches/azuremachinetemplate-azl3-image.yaml - path: patches/cloud-provider-azure-cacertdir.yaml - path: patches/cloud-provider-azure-ci-cacertdir.yaml diff --git a/templates/test/ci/prow-azl3/patches/controller-manager.yaml b/templates/test/ci/prow-azl3/patches/controller-manager.yaml index a064ec5792e..060c4c97e39 100644 --- a/templates/test/ci/prow-azl3/patches/controller-manager.yaml +++ b/templates/test/ci/prow-azl3/patches/controller-manager.yaml @@ -1,15 +1,17 @@ -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - preKubeadmCommands: - - | +- op: add + path: /spec/kubeadmConfigSpec/files/0 + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + # Install ca-certificates packages for Azure Linux tdnf install -y ca-certificates ca-certificates-legacy update-ca-trust - + # Follow Azure Linux 3 docs exactly - completely permissive for debugging # Change default policy to ACCEPT (as recommended by AZL3 docs) iptables -P INPUT ACCEPT @@ -26,3 +28,10 @@ spec: iptables-save > /etc/systemd/scripts/ip4save ip6tables-save > /etc/systemd/scripts/ip6save + path: /tmp/azl3-setup.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/0 + value: + bash -c /tmp/azl3-setup.sh diff --git a/templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml b/templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml index d76a784f62b..42e57439d6b 100644 --- a/templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml +++ b/templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml @@ -1,29 +1,37 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - preKubeadmCommands: - - | - # Install ca-certificates packages for Azure Linux - tdnf install -y ca-certificates ca-certificates-legacy - update-ca-trust - - # Follow Azure Linux 3 docs exactly - completely permissive for debugging - # Change default policy to ACCEPT (as recommended by AZL3 docs) - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT +- op: add + path: /spec/template/spec/files/0 + value: + content: | + #!/bin/bash - ip6tables -P INPUT ACCEPT - ip6tables -P FORWARD ACCEPT - ip6tables -P OUTPUT ACCEPT + set -o nounset + set -o pipefail + set -o errexit - # Flush any rules which would filter packets - iptables -F - ip6tables -F + # Install ca-certificates packages for Azure Linux + tdnf install -y ca-certificates ca-certificates-legacy + update-ca-trust + + # Follow Azure Linux 3 docs exactly - completely permissive for debugging + # Change default policy to ACCEPT (as recommended by AZL3 docs) + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT - iptables-save > /etc/systemd/scripts/ip4save - ip6tables-save > /etc/systemd/scripts/ip6save + ip6tables -P INPUT ACCEPT + ip6tables -P FORWARD ACCEPT + ip6tables -P OUTPUT ACCEPT + + # Flush any rules which would filter packets + iptables -F + ip6tables -F + + iptables-save > /etc/systemd/scripts/ip4save + ip6tables-save > /etc/systemd/scripts/ip6save + path: /tmp/azl3-setup.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/0 + value: + bash -c /tmp/azl3-setup.sh diff --git a/templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml b/templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml new file mode 100644 index 00000000000..c44ae2bdf32 --- /dev/null +++ b/templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml @@ -0,0 +1,19 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: + marketplace: null +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + marketplace: null diff --git a/templates/test/ci/prow-ci-version-azl3/kustomization.yaml b/templates/test/ci/prow-ci-version-azl3/kustomization.yaml new file mode 100644 index 00000000000..eb46c9d12a5 --- /dev/null +++ b/templates/test/ci/prow-ci-version-azl3/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../prow-ci-version +patches: +- path: ../prow-azl3/patches/controller-manager.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: ../prow-azl3/patches/kubeadm-config-template-azl3.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: ../prow-azl3/patches/azuremachinetemplate-azl3-image.yaml +- path: ../prow-azl3/patches/remove-marketplace-image.yaml +- path: ../prow-azl3/patches/cloud-provider-azure-cacertdir.yaml +- path: ../prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml + +sortOptions: + order: fifo diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 32efdf03803..1ac239b019f 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -52,8 +52,6 @@ type AzureLogCollector struct{} const ( collectLogInterval = 3 * time.Second collectLogTimeout = 1 * time.Minute - // extendedCollectLogTimeout is used for scenarios where nodes might take longer to be ready (e.g., additional control plane nodes) - extendedCollectLogTimeout = 3 * time.Minute ) var _ framework.ClusterLogCollector = &AzureLogCollector{} @@ -336,20 +334,13 @@ func collectLogsFromNode(cluster *clusterv1.Cluster, hostname string, isWindows execToPathFn := func(outputFileName, command string, args ...string) func() error { return func() error { - // Use extended timeout for better resilience, especially for additional control plane nodes - return retryWithTimeout(collectLogInterval, extendedCollectLogTimeout, func() error { + return retryWithTimeout(collectLogInterval, collectLogTimeout, func() error { f, err := fileOnHost(filepath.Join(outputPath, outputFileName)) if err != nil { - Logf("Failed to create output file %s for node %s: %v", outputFileName, hostname, err) return err } defer f.Close() - - err = execOnHost(controlPlaneEndpoint, hostname, sshPort, extendedCollectLogTimeout, f, command, args...) - if err != nil { - Logf("Failed to execute command '%s %v' on node %s: %v", command, args, hostname, err) - } - return err + return execOnHost(controlPlaneEndpoint, hostname, sshPort, collectLogTimeout, f, command, args...) }) } } @@ -426,10 +417,6 @@ func getAzureMachinePool(ctx context.Context, managementClusterClient client.Cli func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { return []func() error{ - execToPathFn( - "ssh-connectivity-test.txt", - "echo", "SSH connectivity test successful - $(date) - hostname: $(hostname) - uptime: $(uptime)", - ), execToPathFn( "journal.log", "sudo", "journalctl", "--no-pager", "--output=short-precise", @@ -524,10 +511,6 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ... "kube-apiserver.log", crictlPodLogsCmd("kube-apiserver"), ), - execToPathFn( - "log-collection-summary.txt", - "echo", "Log collection completed at $(date) for node $(hostname). SSH connectivity was successful if you can see this message.", - ), } } diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 2b16516ad9f..bfb2fc3b984 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -224,15 +224,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) - // By("Verifying expected VM extensions are present on the node", func() { - // AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { - // return AzureVMExtensionsSpecInput{ - // BootstrapClusterProxy: bootstrapClusterProxy, - // Namespace: namespace, - // ClusterName: clusterName, - // } - // }) - // }) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) By("Verifying security rules are deleted on azure side", func() { AzureSecurityGroupsSpec(ctx, func() AzureSecurityGroupsSpecInput { @@ -293,7 +293,7 @@ var _ = Describe("Workload cluster creation", func() { withFlavor("azure-cni-v1"), withNamespace(namespace.Name), withClusterName(clusterName), - withControlPlaneMachineCount(1), + withControlPlaneMachineCount(3), withWorkerMachineCount(2), withControlPlaneInterval(specName, "wait-control-plane-ha"), withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ @@ -310,15 +310,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) - // By("can expect VM extensions are present on the node", func() { - // AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { - // return AzureVMExtensionsSpecInput{ - // BootstrapClusterProxy: bootstrapClusterProxy, - // Namespace: namespace, - // ClusterName: clusterName, - // } - // }) - // }) + By("can expect VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) By("can validate failure domains", func() { AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 58a4c59d815..4ebbd186bd3 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -148,6 +148,8 @@ providers: targetName: "cluster-template-conformance-presubmit-artifacts-dra.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-dra.yaml" targetName: "cluster-template-conformance-ci-artifacts-dra.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml" + targetName: "cluster-template-conformance-ci-artifacts-azl3.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml" targetName: "cluster-template-machine-pool-flex.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml" diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index f6d3442fc4c..8eaa0544702 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -116,6 +116,9 @@ var _ = Describe("Conformance Tests", func() { } } + // TODO: Remove this as it's for testing + flavor = "conformance-ci-artifacts-azl3" + // Starting with Kubernetes v1.25, the kubetest config file needs to be compatible with Ginkgo V2. v125 := semver.MustParse("1.25.0-alpha.0.0") v, err := semver.ParseTolerant(kubernetesVersion) From f1ccea4904b85e84ed68c32daea564da27e6a338 Mon Sep 17 00:00:00 2001 From: William Yao Date: Mon, 25 Aug 2025 18:49:11 -0700 Subject: [PATCH 17/19] Try 1.33.2 temproarily --- templates/test/ci/cluster-template-prow-azl3.yaml | 4 ++-- templates/test/ci/cluster-template-prow-ci-version-azl3.yaml | 4 ++-- .../ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-azl3.yaml b/templates/test/ci/cluster-template-prow-azl3.yaml index 9eac07241e7..04fd1747314 100644 --- a/templates/test/ci/cluster-template-prow-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-azl3.yaml @@ -165,7 +165,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 osDisk: diskSizeGB: 128 osType: Linux @@ -213,7 +213,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml index 71769fe589f..e0c53cae1c3 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml @@ -271,7 +271,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 osDisk: diskSizeGB: 128 osType: Linux @@ -321,7 +321,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml index 143e4be2dfc..a29c5dd88da 100644 --- a/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml +++ b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml @@ -9,7 +9,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate @@ -22,4 +22,4 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: ${KUBERNETES_VERSION#v} + version: 1.33.2 From 94e1b203500884890c4417dcc5ae9ccf02ce32f9 Mon Sep 17 00:00:00 2001 From: William Yao Date: Tue, 26 Aug 2025 10:38:55 -0700 Subject: [PATCH 18/19] Add separate test target --- .../test/ci/cluster-template-prow-azl3.yaml | 4 +- ...cluster-template-prow-ci-version-azl3.yaml | 4 +- .../azuremachinetemplate-azl3-image.yaml | 4 +- test/e2e/azure_logcollector.go | 48 ----------- test/e2e/azure_test.go | 85 +++++++++++++++++++ test/e2e/conformance_test.go | 7 +- 6 files changed, 94 insertions(+), 58 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-azl3.yaml b/templates/test/ci/cluster-template-prow-azl3.yaml index 04fd1747314..79b7b13d183 100644 --- a/templates/test/ci/cluster-template-prow-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-azl3.yaml @@ -165,7 +165,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} osDisk: diskSizeGB: 128 osType: Linux @@ -213,7 +213,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml index e0c53cae1c3..1834da62ce2 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml @@ -271,7 +271,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} osDisk: diskSizeGB: 128 osType: Linux @@ -321,7 +321,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} osDisk: diskSizeGB: 128 osType: Linux diff --git a/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml index a29c5dd88da..d664fda1d9f 100644 --- a/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml +++ b/templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml @@ -9,7 +9,7 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate @@ -22,4 +22,4 @@ spec: computeGallery: gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019 name: capi-azurelinux-3 - version: 1.33.2 + version: ${AZL3_VERSION:="1.33.2"} diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 1ac239b019f..95b99d4ebf0 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -449,58 +449,10 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ... "cloud-init-output.log", "sudo", "sh", "-c", "echo 'Waiting for cloud-init to complete before collecting output log...' && cloud-init status --wait && echo 'Cloud-init completed, collecting output log...' && if [ -f /var/log/cloud-init-output.log ]; then echo 'Found cloud-init-output.log, reading contents:' && sudo cat /var/log/cloud-init-output.log; else echo 'cloud-init-output.log not found after cloud-init completion'; fi", ), - execToPathFn( - "cloud-init-journal.log", - "sudo", "journalctl", "--no-pager", "--output=short-precise", "-u", "cloud-init", "-u", "cloud-config", "-u", "cloud-final", - ), - execToPathFn( - "cloud-init-status.txt", - "sudo", "cloud-init", "status", "--long", - ), - execToPathFn( - "cloud-init-all-logs.txt", - "sudo", "sh", "-c", "echo '=== cloud-init logs from journal ===' && journalctl --no-pager -u cloud-init-local -u cloud-init -u cloud-config -u cloud-final --output=short-precise && echo && echo '=== cloud-init result.json ===' && cat /run/cloud-init/result.json 2>/dev/null || echo 'result.json not found' && echo && echo '=== cloud-init instance-data.json ===' && cat /run/cloud-init/instance-data.json 2>/dev/null || echo 'instance-data.json not found'", - ), - execToPathFn( - "cloud-init-file-details.txt", - "sudo", "sh", "-c", "echo '=== Cloud-init file existence and permissions ===' && echo 'Timestamp: '$(date) && echo 'Cloud-init status:' && cloud-init status && echo && echo 'Files in /var/log/ matching cloud-init*:' && ls -la /var/log/cloud-init* 2>/dev/null || echo 'No cloud-init files found in /var/log/' && echo && echo 'Files in /run/cloud-init/:' && ls -la /run/cloud-init/ 2>/dev/null || echo '/run/cloud-init/ not found' && echo && echo 'Checking for sentinel file:' && ls -la /run/cluster-api/ 2>/dev/null || echo '/run/cluster-api/ not found' && echo && echo 'SELinux context (if applicable):' && ls -laZ /var/log/cloud-init* 2>/dev/null || echo 'No SELinux or cloud-init files'", - ), - execToPathFn( - "cloud-init-output-comprehensive.log", - "sudo", "sh", "-c", "echo '=== Comprehensive cloud-init output collection ===' && echo 'Method 1: Direct sudo cat:' && sudo cat /var/log/cloud-init-output.log", - ), - execToPathFn( - "cloud-init-output-methods.log", - "sudo", "sh", "-c", "echo 'Method 2: sudo tail:' && sudo tail -n +1 /var/log/cloud-init-output.log && echo && echo 'Method 3: sudo dd:' && sudo dd if=/var/log/cloud-init-output.log 2>/dev/null && echo && echo 'Method 4: File readability test:' && sudo test -r /var/log/cloud-init-output.log && echo 'File readable with sudo' || echo 'File not readable with sudo'", - ), - execToPathFn( - "cloud-init-userdata.log", - "sudo", "cloud-init", "query", "userdata", - ), execToPathFn( "sentinel-file-dir.txt", "ls", "-la", "/run/cluster-api/", ), - execToPathFn( - "var-log-dir.txt", - "ls", "-la", "/var/log/", - ), - execToPathFn( - "system-info.txt", - "sudo", "sh", "-c", "echo '=== OS Release ===' && cat /etc/os-release && echo && echo '=== Uptime ===' && uptime && echo && echo '=== Free memory ===' && free -h", - ), - execToPathFn( - "dmesg.log", - "sudo", "dmesg", - ), - execToPathFn( - "systemd-analyze.txt", - "sudo", "systemd-analyze", "blame", - ), - execToPathFn( - "cloud-init-run-dir.txt", - "sudo", "ls", "-la", "/run/cloud-init/", - ), execToPathFn( "cni.log", "cat", "/var/log/calico/cni/cni.log", diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index bfb2fc3b984..1de83bbc181 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -1295,5 +1295,90 @@ var _ = Describe("Workload cluster creation", func() { }) }) + Context("Creating a highly-available cluster with Azure Linux 3 [OPTIONAL]", func() { + It("with three controlplane node and two worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "azl3") + + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withNamespace(namespace.Name), + withClusterName(clusterName), + withFlavor("azl3"), + withControlPlaneMachineCount(3), + withWorkerMachineCount(2), + withControlPlaneInterval(specName, "wait-control-plane-ha"), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + }), + withPostMachinesProvisioned(func() { + EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { + return DaemonsetsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }), + ), result) + + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + + By("Verifying security rules are deleted on azure side", func() { + AzureSecurityGroupsSpec(ctx, func() AzureSecurityGroupsSpecInput { + return AzureSecurityGroupsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + Cluster: result.Cluster, + WaitForUpdate: e2eConfig.GetIntervals(specName, "wait-nsg-update"), + } + }) + }) + + By("Validating failure domains", func() { + AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { + return AzureFailureDomainsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Cluster: result.Cluster, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + + By("Creating an accessible load balancer", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + By("Validating network policies", func() { + AzureNetPolSpec(ctx, func() AzureNetPolSpecInput { + return AzureNetPolSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + By("PASSED!") + }) + }) + // TODO: add a same test as above for a windows cluster }) diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 8eaa0544702..60c32364fb0 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -102,7 +102,9 @@ var _ = Describe("Conformance Tests", func() { if flavor == "" { if useCIArtifacts { - flavor = "conformance-ci-artifacts" + // flavor = "conformance-ci-artifacts" + // TODO: Remove this as it's for testing + flavor = "conformance-ci-artifacts-azl3" } else if usePRArtifacts { flavor = "conformance-presubmit-artifacts" } @@ -116,9 +118,6 @@ var _ = Describe("Conformance Tests", func() { } } - // TODO: Remove this as it's for testing - flavor = "conformance-ci-artifacts-azl3" - // Starting with Kubernetes v1.25, the kubetest config file needs to be compatible with Ginkgo V2. v125 := semver.MustParse("1.25.0-alpha.0.0") v, err := semver.ParseTolerant(kubernetesVersion) From 08c9c11492ef549dca7802c2e6a9d33aa8e6e997 Mon Sep 17 00:00:00 2001 From: William Yao Date: Tue, 26 Aug 2025 16:03:31 -0700 Subject: [PATCH 19/19] Revert hardcode --- test/e2e/azure_test.go | 1 + test/e2e/conformance_test.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 1de83bbc181..4a9c0031311 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -1298,6 +1298,7 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating a highly-available cluster with Azure Linux 3 [OPTIONAL]", func() { It("with three controlplane node and two worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "azl3") + Expect(os.Setenv("AZL3_VERSION", "${KUBERNETES_VERSION}")).To(Succeed()) clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 60c32364fb0..f6d3442fc4c 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -102,9 +102,7 @@ var _ = Describe("Conformance Tests", func() { if flavor == "" { if useCIArtifacts { - // flavor = "conformance-ci-artifacts" - // TODO: Remove this as it's for testing - flavor = "conformance-ci-artifacts-azl3" + flavor = "conformance-ci-artifacts" } else if usePRArtifacts { flavor = "conformance-presubmit-artifacts" }