Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

enhance: support network isolation for tencentcloud clb plugin #183

Merged
merged 1 commit into from
Nov 12, 2024
Merged

enhance: support network isolation for tencentcloud clb plugin #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 128 additions & 70 deletions cloudprovider/tencentcloud/clb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const (
MinPortConfigName = "MinPort"
MaxPortConfigName = "MaxPort"
OwnerPodKey = "game.kruise.io/owner-pod"
TargetPortKey = "game.kruise.io/target-port"
)

type portAllocated map[int32]bool
Expand Down Expand Up @@ -109,6 +110,29 @@ func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Con
return pod, nil
}

func (p *ClbPlugin) deleteListener(ctx context.Context, c client.Client, lis *v1alpha1.DedicatedCLBListener) cperrors.PluginError {
err := c.Delete(ctx, lis)
if err != nil {
return cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
if pm := p.cache[lis.Spec.LbId]; pm != nil {
pm[int32(lis.Spec.LbPort)] = false
}
var podName string
if targetPod := lis.Spec.TargetPod; targetPod != nil {
podName = targetPod.PodName
} else if lis.Labels != nil && lis.Labels[TargetPortKey] != "" && lis.Labels[OwnerPodKey] != "" {
podName = lis.Labels[OwnerPodKey]
} else {
return nil
}
target := fmt.Sprintf("%s/%d", lis.Spec.LbId, lis.Spec.LbPort)
p.podAllocate[podName] = slices.DeleteFunc(p.podAllocate[podName], func(el string) bool {
return el == target
})
return nil
}

func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
if pod.DeletionTimestamp != nil {
return pod, nil
Expand Down Expand Up @@ -159,27 +183,38 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
return pod, nil
}

// delete dedicated clb listener if necessary (target pod not match)
targetPod := lis.Spec.TargetPod
if targetPod == nil || targetPod.PodName != pod.Name {
err := c.Delete(ctx, &lis)
if targetPod != nil && targetPod.PodName == pod.Name {
port := portProtocol{
port: int(targetPod.TargetPort),
protocol: lis.Spec.Protocol,
}
lisMap[port] = lis
} else if targetPod == nil && (lis.Labels != nil && lis.Labels[TargetPortKey] != "") {
targetPort, err := strconv.Atoi(lis.Labels[TargetPortKey])
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
log.Warningf("[%s] invalid dedicated clb listener target port annotation %s/%s: %s", ClbNetwork, lis.Namespace, lis.Name, err.Error())
continue
}
port := portProtocol{
port: targetPort,
protocol: lis.Spec.Protocol,
}
// lower priority than targetPod is not nil
if _, exists := lisMap[port]; !exists {
lisMap[port] = lis
}
continue
}
port := portProtocol{
port: int(targetPod.TargetPort),
protocol: lis.Spec.Protocol,
}
lisMap[port] = lis
}

internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)

for _, port := range clbConf.targetPorts {
if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one
if networkManager.GetNetworkDisabled() {
continue
}
// ensure not ready while creating the listener
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
Expand All @@ -197,67 +232,89 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
}
} else { // already created dedicated clb listener bound to pod
delete(lisMap, port)
// recreate dedicated clb listener if necessary (config changed)
if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) {
// ensure not ready while recreating the listener
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}

// delete old listener
err := c.Delete(ctx, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}

// allocate and create new listener bound to pod
newLis, err := p.consLis(clbConf, pod, port, gss.Name)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
err = c.Create(ctx, newLis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
if networkManager.GetNetworkDisabled() { // disable network
// deregister pod if networkDisabled is true
if lis.Spec.TargetPod != nil {
lis.Spec.TargetPod = nil
err = c.Update(ctx, &lis)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
}
}
} else { // dedicated clb listener is desired, check status
if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready
ss := strings.Split(lis.Status.Address, ":")
if len(ss) != 2 {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address))
} else { // enable network
if lis.Spec.TargetPod == nil { // ensure target pod is bound to dedicated clb listener
lis.Spec.TargetPod = &v1alpha1.TargetPod{
PodName: pod.Name,
TargetPort: int64(port.port),
}
lbPort, err := strconv.Atoi(ss[1])
err = c.Update(ctx, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1]))
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
}
instrIPort := intstr.FromInt(int(port.port))
instrEPort := intstr.FromInt(lbPort)
internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
IP: pod.Status.PodIP,
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrIPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
IP: ss[0],
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrEPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
} else { // network not ready
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
} else {
// recreate dedicated clb listener if necessary (config changed)
if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) {
// ensure not ready while recreating the listener
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}

// delete old listener
err := p.deleteListener(ctx, c, &lis)
if err != nil {
return pod, err
}

// allocate and create new listener bound to pod
if newLis, err := p.consLis(clbConf, pod, port, gss.Name); err != nil {
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
} else {
err := c.Create(ctx, newLis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
}
} else { // dedicated clb listener is desired, check status
if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready
ss := strings.Split(lis.Status.Address, ":")
if len(ss) != 2 {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address))
}
lbPort, err := strconv.Atoi(ss[1])
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1]))
}
instrIPort := intstr.FromInt(int(port.port))
instrEPort := intstr.FromInt(lbPort)
internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
IP: pod.Status.PodIP,
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrIPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
IP: ss[0],
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrEPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
} else { // network not ready
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
}
}
}
}
Expand All @@ -266,9 +323,9 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C

// other dedicated clb listener is not used, delete it
for _, lis := range lisMap {
err := c.Delete(ctx, &lis)
err := p.deleteListener(ctx, c, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
return pod, err
}
}

Expand Down Expand Up @@ -304,7 +361,8 @@ func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtoc
GenerateName: pod.Name + "-",
Namespace: pod.Namespace,
Labels: map[string]string{
OwnerPodKey: pod.Name,
OwnerPodKey: pod.Name, // used to select pod related dedicated clb listener
TargetPortKey: strconv.Itoa(port.port), // used to recover clb pod binding when networkDisabled set from true to false
kruisev1alpha1.GameServerOwnerGssKey: gssName,
},
OwnerReferences: []metav1.OwnerReference{
Expand Down
5 changes: 2 additions & 3 deletions docs/en/user_manuals/network.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -985,10 +985,9 @@ TencentCloud

#### Plugin description


- TencentCloud-CLB enables game servers to be accessed from the Internet by using Cloud Load Balancer (CLB) of Tencent Cloud. CLB is a type of Server Load Balancer (CLB). TencentCloud-CLB uses different ports for different game servers. The CLB instance only forwards traffic, but does not implement load balancing.
The [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) network plugin needs to be installed (can be installed through the TKE application market).
- This network plugin does not support network isolation.
- The [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) network plugin needs to be installed (can be installed through the TKE application market).
- This network plugin supports network isolation.

#### Network parameters

Expand Down
2 changes: 1 addition & 1 deletion docs/中文/用户手册/网络模型.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -988,7 +988,7 @@ TencentCloud

- TencentCloud-CLB 使用腾讯云负载均衡器（CLB）作为对外服务的承载实体，在此模式下，不同游戏服使用 CLB 的不同端口对外暴露，此时 CLB 只做转发，并未均衡流量。
- 需安装 [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) 网络插件(可通过 TKE 应用市场安装)。
- 是否支持网络隔离：否。
- 是否支持网络隔离：是。

#### 网络参数

Expand Down
Loading