Add pre-kubeadm join commands to worker nodes

- Also pre-create a DNS Name for the frontend LB's IP that get assigned to the controlplane VM. - precreate internal LB for the API server and set default internal IP for it. - update templates with new DNS name for the API Server. - Update /etc/hosts on the worker nodes with internal IP and DNS name of the public LB.
kubernetes-sigs · Oct 28, 2024 · 7734183 · 7734183
1 parent 9ba44ee
commit 7734183
Show file tree

Hide file tree

Showing 21 changed files with 192 additions and 20 deletions.
diff --git a/api/v1beta1/tags.go b/api/v1beta1/tags.go
@@ -119,6 +119,10 @@ const (
 	// APIServerRole describes the value for the apiserver role.
 	APIServerRole = "apiserver"
 
+	// APIServerRoleInternal describes the value for the apiserver-internal role,
+	// an identifier for an internal load balancer serving apiserver traffic for cluster nodes.
+	APIServerRoleInternal = "apiserver-internal"
+
 	// NodeOutboundRole describes the value for the node outbound LB role.
 	NodeOutboundRole = "nodeOutbound"
 

diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go
@@ -265,6 +265,35 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
 		},
 	}
 
+	if s.APIServerLB().Type != infrav1.Internal {
+		specs = append(specs, &loadbalancers.LBSpec{
+			Name:              s.APIServerLB().Name + "-ilb",
+			ResourceGroup:     s.ResourceGroup(),
+			SubscriptionID:    s.SubscriptionID(),
+			ClusterName:       s.ClusterName(),
+			Location:          s.Location(),
+			ExtendedLocation:  s.ExtendedLocation(),
+			VNetName:          s.Vnet().Name,
+			VNetResourceGroup: s.Vnet().ResourceGroup,
+			SubnetName:        s.ControlPlaneSubnet().Name,
+			FrontendIPConfigs: []infrav1.FrontendIP{
+				{
+					Name: s.APIServerLB().Name + "-ilb-frontEnd",
+					FrontendIPClass: infrav1.FrontendIPClass{
+						PrivateIPAddress: infrav1.DefaultInternalLBIPAddress,
+					},
+				},
+			},
+			APIServerPort:        s.APIServerPort(),
+			Type:                 infrav1.Internal,
+			SKU:                  s.APIServerLB().SKU,
+			Role:                 infrav1.APIServerRoleInternal,
+			BackendPoolName:      s.APIServerLB().BackendPool.Name + "-ilb",
+			IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes,
+			AdditionalTags:       s.AdditionalTags(),
+		})
+	}
+
 	// Node outbound LB
 	if s.NodeOutboundLB() != nil {
 		specs = append(specs, &loadbalancers.LBSpec{

diff --git a/azure/scope/machine.go b/azure/scope/machine.go
@@ -294,10 +294,9 @@ func (m *MachineScope) BuildNICSpec(nicName string, infrav1NetworkInterface infr
 		if m.Role() == infrav1.ControlPlane {
 			spec.PublicLBName = m.OutboundLBName(m.Role())
 			spec.PublicLBAddressPoolName = m.OutboundPoolName(m.Role())
-			if m.IsAPIServerPrivate() {
-				spec.InternalLBName = m.APIServerLBName()
-				spec.InternalLBAddressPoolName = m.APIServerLBPoolName()
-			} else {
+			spec.InternalLBName = m.APIServerLBName() + "-ilb"
+			spec.InternalLBAddressPoolName = m.APIServerLBPoolName() + "-ilb"
+			if !m.IsAPIServerPrivate() {
 				spec.PublicLBNATRuleName = m.Name()
 				spec.PublicLBAddressPoolName = m.APIServerLBPoolName()
 			}

diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go
@@ -73,7 +73,6 @@ func (s *Service) Reconcile(ctx context.Context) error {
 
 	ctx, cancel := context.WithTimeout(ctx, s.Scope.DefaultedAzureServiceReconcileTimeout())
 	defer cancel()
-
 	specs := s.Scope.LBSpecs()
 	if len(specs) == 0 {
 		return nil

diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go
@@ -214,7 +214,7 @@ func getOutboundRules(lbSpec LBSpec, frontendIDs []*armnetwork.SubResource) []*a
 }
 
 func getLoadBalancingRules(lbSpec LBSpec, frontendIDs []*armnetwork.SubResource) []*armnetwork.LoadBalancingRule {
-	if lbSpec.Role == infrav1.APIServerRole {
+	if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.APIServerRoleInternal {
 		// We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule.
 		// For more information on Standard LB outbound connections see https://learn.microsoft.com/azure/load-balancer/load-balancer-outbound-connections.
 		var frontendIPConfig *armnetwork.SubResource
@@ -255,7 +255,7 @@ func getBackendAddressPools(lbSpec LBSpec) []*armnetwork.BackendAddressPool {
 }
 
 func getProbes(lbSpec LBSpec) []*armnetwork.Probe {
-	if lbSpec.Role == infrav1.APIServerRole {
+	if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.APIServerRoleInternal {
 		return []*armnetwork.Probe{
 			{
 				Name: ptr.To(httpsProbe),

diff --git a/scripts/aks-as-mgmt.sh b/scripts/aks-as-mgmt.sh
@@ -24,9 +24,8 @@ source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI
 source "${REPO_ROOT}/hack/ensure-tags.sh" # set the right timestamp and job name
 
 KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
-KIND="${REPO_ROOT}/hack/tools/bin/kind"
 AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
-make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" "${AZWI##*/}"
+make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${AZWI##*/}"
 
 export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # management cluster name
 export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # resource group name
@@ -48,6 +47,15 @@ export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}"
 export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}"
 export AZURE_TENANT_ID="${AZURE_TENANT_ID:-}"
 
+# to suppress unbound variable error message
+export APISERVER_LB_DNS_SUFFIX="${APISERVER_LB_DNS_SUFFIX:-}"
+export AKS_MI_CLIENT_ID="${AKS_MI_CLIENT_ID:-}"
+export AKS_MI_OBJECT_ID="${AKS_MI_OBJECT_ID:-}"
+export AKS_MI_RESOURCE_ID="${AKS_MI_RESOURCE_ID:-}"
+export MANAGED_IDENTITY_NAME="${MANAGED_IDENTITY_NAME:-}"
+export MANAGED_IDENTITY_RG="${MANAGED_IDENTITY_RG:-}"
+export ASO_CREDENTIAL_SECRET_MODE="${ASO_CREDENTIAL_SECRET_MODE:-}"
+
 main() {
 
   echo "--------------------------------"
@@ -158,6 +166,13 @@ create_aks_cluster() {
 
   echo "using ASO_CREDENTIAL_SECRET_MODE as podidentity"
   ASO_CREDENTIAL_SECRET_MODE="podidentity"
+
+  echo "creating APISERVER_LB_DNS_SUFFIX for the frontend IP of the cluster"
+  # ensure that the APISERVER_LB_DNS_SUFFIX is of length 10 and contains only alphanumeric characters
+  # LC_ALL=C is used to set the locale to C to ensure that the output is ASCII
+  # head /dev/urandom generates random bytes. Will work on Linux and MacOS. Also works on Windows with WSL.
+  APISERVER_LB_DNS_SUFFIX=$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 10)
+  export APISERVER_LB_DNS_SUFFIX
 }
 
 set_env_varaibles(){
@@ -178,6 +193,7 @@ kustomize_substitutions:
   CLUSTER_IDENTITY_TYPE: "UserAssignedMSI"
   ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
   REGISTRY: "${REGISTRY}"
+  APISERVER_LB_DNS_SUFFIX: "${APISERVER_LB_DNS_SUFFIX}"
 allowed_contexts:
   - "$MGMT_CLUSTER_NAME"
   - "kind-capz"

diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml
diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml
diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml
diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml
diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml
diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -29,6 +29,12 @@ spec:
     name: ${CLUSTER_IDENTITY_NAME}
   location: ${AZURE_LOCATION}
   networkSpec:
+    apiServerLB:
+      frontendIPs:
+      - name: ${CLUSTER_NAME}-api-lb
+        publicIP:
+          dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com
+          name: ${CLUSTER_NAME}-api-lb
     subnets:
     - name: control-plane-subnet
       role: control-plane
@@ -189,7 +195,9 @@ spec:
           kubeletExtraArgs:
             cloud-provider: external
           name: '{{ ds.meta_data["local_hostname"] }}'
-      preKubeadmCommands: []
+      preKubeadmCommands:
+      - echo '10.0.0.100   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'
+        >> /etc/hosts
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: AzureClusterIdentity

diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml
@@ -5,8 +5,11 @@ resources:
 - ../base
 - machine-deployment.yaml
 - ../../azure-cluster-identity
+
 patches:
 - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml
+- path: patches/control-plane.yaml
+- path: patches/kubeadm-config-template.yaml
 
 sortOptions:
   order: fifo
diff --git a/templates/flavors/default/patches/control-plane.yaml b/templates/flavors/default/patches/control-plane.yaml
@@ -0,0 +1,15 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  networkSpec:
+    apiServerLB:
+      # Hack: We pre-create this public IP and the DNS name to use it in the
+      # worker node's /etc/hosts. This resolves the routing issue due to newly
+      # implemented security rules in Azure.
+      frontendIPs:
+        - name: ${CLUSTER_NAME}-api-lb
+          publicIP:
+            name: ${CLUSTER_NAME}-api-lb
+            dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com
diff --git a/templates/flavors/default/patches/kubeadm-config-template.yaml b/templates/flavors/default/patches/kubeadm-config-template.yaml
@@ -0,0 +1,12 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      # Hack: /etc/hosts file is updated with a pre-created DNS name of the API server and internal LB
+      # so that worker nodes can access the API server using the internal IP.
+      # 10.0.0.100 is the default IP that gets assigned to a internal load balancer.
+      preKubeadmCommands:
+        - echo '10.0.0.100   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml
@@ -7,6 +7,7 @@ resources:
 
 patches:
 - path: ../base-windows-containerd/cluster.yaml
+- path: patches/kubeadm-config-template.yaml
 
 sortOptions:
   order: fifo
diff --git a/templates/flavors/windows/patches/kubeadm-config-template.yaml b/templates/flavors/windows/patches/kubeadm-config-template.yaml
@@ -0,0 +1,12 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-win"
+spec:
+  template:
+    spec:
+      # Hack: /etc/hosts file is updated with a pre-created DNS name of the API server and internal LB
+      # so that worker nodes can access the API server using the internal IP.
+      # 10.0.0.100 is the default IP that gets assigned to a internal load balancer.
+      preKubeadmCommands:
+        - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.0.0.100   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml