Skip to content

Commit e591f24

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #5209 from nawazkh/update_kubeadm_configs
Use internal LB's IP for intra-node communication
2 parents 34895ef + 25bca8d commit e591f24

23 files changed

+239
-24
lines changed

‎Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,12 @@ export CI_RG ?= $(AZWI_RESOURCE_GROUP)
208208
export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG))
209209
export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id
210210

211+
# ensure that the APISERVER_LB_DNS_SUFFIX is of length 10 and contains only alphanumeric characters
212+
# LC_ALL=C is used to set the locale to C to ensure that the output is ASCII
213+
# head /dev/urandom generates random bytes. Will work on Linux and MacOS. Also works on Windows with WSL.
214+
# Ignore SIGPIPE error. This will suppress error messages if head closes the pipe before tr finishes.
215+
export APISERVER_LB_DNS_SUFFIX := $(shell LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 10 2>/dev/null)
216+
211217
## --------------------------------------
212218
## Binaries
213219
## --------------------------------------

‎api/v1beta1/tags.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ const (
119119
// APIServerRole describes the value for the apiserver role.
120120
APIServerRole = "apiserver"
121121

122+
// APIServerRoleInternal describes the value for the apiserver-internal role,
123+
// an identifier for an internal load balancer serving apiserver traffic for cluster nodes.
124+
APIServerRoleInternal = "apiserver-internal"
125+
122126
// NodeOutboundRole describes the value for the node outbound LB role.
123127
NodeOutboundRole = "nodeOutbound"
124128

‎azure/scope/cluster.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,35 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
265265
},
266266
}
267267

268+
if s.APIServerLB().Type != infrav1.Internal {
269+
specs = append(specs, &loadbalancers.LBSpec{
270+
Name: s.APIServerLB().Name + "-internal",
271+
ResourceGroup: s.ResourceGroup(),
272+
SubscriptionID: s.SubscriptionID(),
273+
ClusterName: s.ClusterName(),
274+
Location: s.Location(),
275+
ExtendedLocation: s.ExtendedLocation(),
276+
VNetName: s.Vnet().Name,
277+
VNetResourceGroup: s.Vnet().ResourceGroup,
278+
SubnetName: s.ControlPlaneSubnet().Name,
279+
FrontendIPConfigs: []infrav1.FrontendIP{
280+
{
281+
Name: s.APIServerLB().Name + "-internal-frontEnd", // TODO: improve this name.
282+
FrontendIPClass: infrav1.FrontendIPClass{
283+
PrivateIPAddress: infrav1.DefaultInternalLBIPAddress,
284+
},
285+
},
286+
},
287+
APIServerPort: s.APIServerPort(),
288+
Type: infrav1.Internal,
289+
SKU: s.APIServerLB().SKU,
290+
Role: infrav1.APIServerRoleInternal,
291+
BackendPoolName: s.APIServerLB().BackendPool.Name + "-internal",
292+
IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes,
293+
AdditionalTags: s.AdditionalTags(),
294+
})
295+
}
296+
268297
// Node outbound LB
269298
if s.NodeOutboundLB() != nil {
270299
specs = append(specs, &loadbalancers.LBSpec{

‎azure/scope/cluster_test.go

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2232,20 +2232,25 @@ func TestBackendPoolName(t *testing.T) {
22322232
}
22332233
clusterScope.AzureCluster.SetBackendPoolNameDefault()
22342234
got := clusterScope.LBSpecs()
2235-
g.Expect(got).To(HaveLen(3))
2235+
g.Expect(got).To(HaveLen(4))
22362236

22372237
// API server backend pool name
22382238
apiServerLBSpec := got[0].(*loadbalancers.LBSpec)
22392239
g.Expect(apiServerLBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName))
22402240
g.Expect(apiServerLBSpec.Role).To(Equal(infrav1.APIServerRole))
22412241

2242+
// API server backend pool name
2243+
apiServerILBSpec := got[1].(*loadbalancers.LBSpec)
2244+
g.Expect(apiServerILBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName + "-internal"))
2245+
g.Expect(apiServerILBSpec.Role).To(Equal(infrav1.APIServerRoleInternal))
2246+
22422247
// Node backend pool name
2243-
NodeLBSpec := got[1].(*loadbalancers.LBSpec)
2248+
NodeLBSpec := got[2].(*loadbalancers.LBSpec)
22442249
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
22452250
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))
22462251

22472252
// Control Plane backend pool name
2248-
controlPlaneLBSpec := got[2].(*loadbalancers.LBSpec)
2253+
controlPlaneLBSpec := got[3].(*loadbalancers.LBSpec)
22492254
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
22502255
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
22512256
})
@@ -2667,6 +2672,33 @@ func TestClusterScope_LBSpecs(t *testing.T) {
26672672
"foo": "bar",
26682673
},
26692674
},
2675+
&loadbalancers.LBSpec{
2676+
Name: "api-server-lb-internal",
2677+
ResourceGroup: "my-rg",
2678+
SubscriptionID: "123",
2679+
ClusterName: "my-cluster",
2680+
Location: "westus2",
2681+
VNetName: "my-vnet",
2682+
VNetResourceGroup: "my-rg",
2683+
SubnetName: "cp-subnet",
2684+
FrontendIPConfigs: []infrav1.FrontendIP{
2685+
{
2686+
Name: "api-server-lb-internal-frontEnd",
2687+
FrontendIPClass: infrav1.FrontendIPClass{
2688+
PrivateIPAddress: infrav1.DefaultInternalLBIPAddress,
2689+
},
2690+
},
2691+
},
2692+
APIServerPort: 6443,
2693+
Type: infrav1.Internal,
2694+
SKU: infrav1.SKUStandard,
2695+
Role: infrav1.APIServerRoleInternal,
2696+
BackendPoolName: "api-server-lb-backend-pool-internal",
2697+
IdleTimeoutInMinutes: ptr.To[int32](30),
2698+
AdditionalTags: infrav1.Tags{
2699+
"foo": "bar",
2700+
},
2701+
},
26702702
&loadbalancers.LBSpec{
26712703
Name: "node-outbound-lb",
26722704
ResourceGroup: "my-rg",

‎azure/scope/machine.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,8 @@ func (m *MachineScope) BuildNICSpec(nicName string, infrav1NetworkInterface infr
298298
spec.InternalLBName = m.APIServerLBName()
299299
spec.InternalLBAddressPoolName = m.APIServerLBPoolName()
300300
} else {
301+
spec.InternalLBName = m.APIServerLBName() + "-internal"
302+
spec.InternalLBAddressPoolName = m.APIServerLBPoolName() + "-internal"
301303
spec.PublicLBNATRuleName = m.Name()
302304
spec.PublicLBAddressPoolName = m.APIServerLBPoolName()
303305
}

‎azure/scope/machine_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2467,8 +2467,8 @@ func TestMachineScope_NICSpecs(t *testing.T) {
24672467
PublicLBName: "api-lb",
24682468
PublicLBAddressPoolName: "api-lb-backendPool",
24692469
PublicLBNATRuleName: "machine-name",
2470-
InternalLBName: "",
2471-
InternalLBAddressPoolName: "",
2470+
InternalLBName: "api-lb-internal",
2471+
InternalLBAddressPoolName: "api-lb-backendPool-internal",
24722472
PublicIPName: "",
24732473
AcceleratedNetworking: nil,
24742474
DNSServers: nil,
@@ -2578,8 +2578,8 @@ func TestMachineScope_NICSpecs(t *testing.T) {
25782578
PublicLBName: "api-lb",
25792579
PublicLBAddressPoolName: "api-lb-backendPool",
25802580
PublicLBNATRuleName: "machine-name",
2581-
InternalLBName: "",
2582-
InternalLBAddressPoolName: "",
2581+
InternalLBName: "api-lb-internal",
2582+
InternalLBAddressPoolName: "api-lb-backendPool-internal",
25832583
PublicIPName: "",
25842584
AcceleratedNetworking: nil,
25852585
DNSServers: []string{"123.123.123.123", "124.124.124.124"},

‎azure/services/loadbalancers/spec.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ func getOutboundRules(lbSpec LBSpec, frontendIDs []*armnetwork.SubResource) []*a
214214
}
215215

216216
func getLoadBalancingRules(lbSpec LBSpec, frontendIDs []*armnetwork.SubResource) []*armnetwork.LoadBalancingRule {
217-
if lbSpec.Role == infrav1.APIServerRole {
217+
if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.APIServerRoleInternal {
218218
// We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule.
219219
// For more information on Standard LB outbound connections see https://learn.microsoft.com/azure/load-balancer/load-balancer-outbound-connections.
220220
var frontendIPConfig *armnetwork.SubResource
@@ -255,7 +255,7 @@ func getBackendAddressPools(lbSpec LBSpec) []*armnetwork.BackendAddressPool {
255255
}
256256

257257
func getProbes(lbSpec LBSpec) []*armnetwork.Probe {
258-
if lbSpec.Role == infrav1.APIServerRole {
258+
if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.APIServerRoleInternal {
259259
return []*armnetwork.Probe{
260260
{
261261
Name: ptr.To(httpsProbe),

‎scripts/aks-as-mgmt.sh

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,11 @@ source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI
2424
source "${REPO_ROOT}/hack/ensure-tags.sh" # set the right timestamp and job name
2525

2626
KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
27-
KIND="${REPO_ROOT}/hack/tools/bin/kind"
2827
AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
29-
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" "${AZWI##*/}"
28+
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${AZWI##*/}"
3029

31-
export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # management cluster name
32-
export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # resource group name
30+
export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # management cluster name
31+
export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # resource group name
3332
export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}"
3433
export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}"
3534
export AZURE_LOCATION="${AZURE_LOCATION:-westus2}"
@@ -48,6 +47,16 @@ export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}"
4847
export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}"
4948
export AZURE_TENANT_ID="${AZURE_TENANT_ID:-}"
5049

50+
# to suppress unbound variable error message
51+
export APISERVER_LB_DNS_SUFFIX="${APISERVER_LB_DNS_SUFFIX:-}"
52+
export AKS_MI_CLIENT_ID="${AKS_MI_CLIENT_ID:-}"
53+
export AKS_MI_OBJECT_ID="${AKS_MI_OBJECT_ID:-}"
54+
export AKS_MI_RESOURCE_ID="${AKS_MI_RESOURCE_ID:-}"
55+
export MANAGED_IDENTITY_NAME="${MANAGED_IDENTITY_NAME:-}"
56+
export MANAGED_IDENTITY_RG="${MANAGED_IDENTITY_RG:-}"
57+
export ASO_CREDENTIAL_SECRET_MODE="${ASO_CREDENTIAL_SECRET_MODE:-}"
58+
export SKIP_AKS_CREATE="${SKIP_AKS_CREATE:-false}"
59+
5160
main() {
5261

5362
echo "--------------------------------"
@@ -66,12 +75,19 @@ main() {
6675
echo "SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH: $SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH"
6776
echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH"
6877
echo "REGISTRY: $REGISTRY"
78+
echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX"
6979

7080
echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID"
7181
echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID"
7282
echo "AZURE_TENANT_ID: $AZURE_TENANT_ID"
7383
echo "--------------------------------"
7484

85+
# if using SKIP_AKS_CREATE=true, skip creating the AKS cluster
86+
if [[ "${SKIP_AKS_CREATE}" == "true" ]]; then
87+
echo "Skipping AKS cluster creation"
88+
return
89+
fi
90+
7591
create_aks_cluster
7692
set_env_varaibles
7793
}
@@ -178,6 +194,7 @@ kustomize_substitutions:
178194
CLUSTER_IDENTITY_TYPE: "UserAssignedMSI"
179195
ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
180196
REGISTRY: "${REGISTRY}"
197+
APISERVER_LB_DNS_SUFFIX: "${APISERVER_LB_DNS_SUFFIX}"
181198
allowed_contexts:
182199
- "$MGMT_CLUSTER_NAME"
183200
- "kind-capz"

‎templates/cluster-template-azure-bastion.yaml

Lines changed: 9 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎templates/cluster-template-azure-cni-v1.yaml

Lines changed: 9 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)