From ab590004163d96cdc4d82a64980496fc73878b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=A9=E8=BD=A9?= Date: Mon, 1 Apr 2024 10:22:58 +0800 Subject: [PATCH] 1. fix the bug of raven l7 is closed due to gateway is deleted 2. fix the bug of raven l3 can not config arp --- cmd/agent/app/options/options.go | 14 +- pkg/engine/proxy.go | 14 +- pkg/networkengine/routedriver/vxlan/vxlan.go | 85 +++++++++-- pkg/networkengine/util/netlink/netlink.go | 25 +++- pkg/networkengine/util/utils.go | 141 ++++++++++++------- 5 files changed, 208 insertions(+), 71 deletions(-) diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 963ddd6..a3c1f78 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -37,6 +37,7 @@ const ( DefaultTunnelMetricsPort = 10265 DefaultProxyMetricsPort = 10266 DefaultHealthyProbeAddr = 10275 + DefaultLocalHost = "127.0.0.1" DefaultMACPrefix = "aa:0f" ) @@ -144,7 +145,7 @@ func (o *AgentOptions) Config() (*config.Config, error) { NodeIP: o.NodeIP, } c.KubeConfig = cfg - c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, c.NodeIP, strconv.Itoa(DefaultTunnelMetricsPort)) + c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, resolveLocalHost(), strconv.Itoa(DefaultTunnelMetricsPort)) c.HealthProbeAddr = resolveAddress(c.HealthProbeAddr, c.NodeIP, strconv.Itoa(DefaultHealthyProbeAddr)) c.Manager, err = newMgr(cfg, c.MetricsBindAddress, c.HealthProbeAddr) if err != nil { @@ -202,7 +203,7 @@ func (o *AgentOptions) Config() (*config.Config, error) { c.Proxy.InternalInsecureAddress = resolveAddress(c.Proxy.InternalInsecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerInsecurePort)) c.Proxy.InternalSecureAddress = resolveAddress(c.Proxy.InternalSecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerSecurePort)) c.Proxy.ExternalAddress = resolveAddress(c.Proxy.ExternalAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerExposedPort)) - c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, c.NodeIP, strconv.Itoa(DefaultProxyMetricsPort)) + c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, resolveLocalHost(), strconv.Itoa(DefaultProxyMetricsPort)) return c, nil } @@ -310,6 +311,15 @@ func getGatewayAPIGroupResource() *restmapper.APIGroupResources { } } +func resolveLocalHost() string { + ipv4Addr, err := net.ResolveIPAddr("ip4", "localhost") + if err != nil { + klog.Warningf("can not get localhost addr, error %s, using default address %s", err.Error(), DefaultLocalHost) + return DefaultLocalHost + } + return ipv4Addr.String() +} + func resolveAddress(srcAddr, defaultHost, defaultPort string) string { if srcAddr == "" { return net.JoinHostPort(defaultHost, defaultPort) diff --git a/pkg/engine/proxy.go b/pkg/engine/proxy.go index 234d88e..932daf5 100644 --- a/pkg/engine/proxy.go +++ b/pkg/engine/proxy.go @@ -97,16 +97,22 @@ func (p *ProxyEngine) processNextWorkItem() bool { } func (p *ProxyEngine) handler(gw *v1beta1.Gateway) error { - proxyStatus := enableProxy(gw) - p.option.SetProxyStatus(proxyStatus) - specServer, specClient := p.getRole(proxyStatus) var err error p.gateway, err = utils.GetOwnGateway(p.client, p.nodeName) if err != nil { klog.Errorf(utils.FormatProxyServer("failed get gateway for %s, can not start proxy server", p.nodeName)) return err } - + proxyStatus := p.option.GetProxyStatus() + if p.gateway != nil && gw.GetName() == p.gateway.GetName() { + proxyStatus = enableProxy(gw) + } else { + if gw.Spec.ExposeType != "" { + proxyStatus = enableProxy(gw) + } + } + p.option.SetProxyStatus(proxyStatus) + specServer, specClient := p.getRole(proxyStatus) switch JudgeType(p.proxyOption.GetServerStatus(), specServer) { case StartType: srcAddr := getSrcAddressForProxyServer(p.client, p.nodeName) diff --git a/pkg/networkengine/routedriver/vxlan/vxlan.go b/pkg/networkengine/routedriver/vxlan/vxlan.go index 0f3111d..0f74483 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan.go @@ -93,9 +93,13 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error var desiredRules, currentRules map[string]*netlink.Rule // The desired and current FDB entries calculated from given network. - // The key is netlink.Neigh.IP + // The key is NeighKey() var desiredFDBs, currentFDBs map[string]*netlink.Neigh + // The desired and current ARP entries calculated from given network. + // The key is NeighKey() + var desiredARPs, currentARPs map[string]*netlink.Neigh + // The desired and current ipset entries calculated from given network. // The key is ip set entry var desiredSet, currentSet map[string]*netlink.IPSetEntry @@ -129,6 +133,11 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error return fmt.Errorf("error listing fdb on node: %s", err) } + currentARPs, err = networkutil.ListARPsOnNode(vx.vxlanIface) + if err != nil { + return fmt.Errorf("error listing arp on node: %s", err) + } + currentSet, err = networkutil.ListIPSetOnNode(vx.ipset) if err != nil { return fmt.Errorf("error listing ip set on node: %s", err) @@ -143,7 +152,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error if err != nil { return fmt.Errorf("error calculate gateway fdb: %s", err) } - + desiredARPs, err = vx.calARPOnGateway(network) + if err != nil { + return fmt.Errorf("error calculate gateway arp: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec) if err != nil { return fmt.Errorf("error deleting non gateway chain rule: %s", err) @@ -158,6 +170,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error if err != nil { return fmt.Errorf("error calculate non gateway fdb: %s", err) } + desiredARPs, err = vx.calARPOnNonGateway(network) + if err != nil { + return fmt.Errorf("error calculate non gateway arp: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec) if err != nil { return fmt.Errorf("error deleting gateway chain rule: %s", err) @@ -180,6 +196,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error if err != nil { return fmt.Errorf("error applying fdb: %s", err) } + err = networkutil.ApplyARPs(currentARPs, desiredARPs) + if err != nil { + return fmt.Errorf("error applying arp: %s", err) + } err = networkutil.ApplyIPSet(vx.ipset, currentSet, desiredSet) if err != nil { return fmt.Errorf("error applying ip set: %s", err) @@ -406,7 +426,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne if err != nil { return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error()) } - fdbs[v.PrivateIP] = &netlink.Neigh{ + nh := &netlink.Neigh{ LinkIndex: vx.vxlanIface.Attrs().Index, State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, Type: netlink.NDA_DST, @@ -415,6 +435,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne IP: net.ParseIP(v.PrivateIP), HardwareAddr: HardwareAddr, } + fdbs[networkutil.NeighKey(nh)] = nh } return fdbs, nil } @@ -428,17 +449,61 @@ func (vx *vxlan) calFDBOnNonGateway(network *types.Network) (map[string]*netlink if err != nil { return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error()) } - return map[string]*netlink.Neigh{ - network.LocalEndpoint.PrivateIP: { + nh := &netlink.Neigh{ + LinkIndex: vx.vxlanIface.Attrs().Index, + State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, + Type: netlink.NDA_DST, + Family: syscall.AF_BRIDGE, + Flags: netlink.NTF_SELF, + IP: net.ParseIP(network.LocalEndpoint.PrivateIP), + HardwareAddr: HardwareAddr, + } + return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil +} + +// calARPOnGateway calculates and returns the desired ARP entries on gateway node. +// The ARP entries format are equivalent to the following `ip neigh` command: +func (vx *vxlan) calARPOnGateway(network *types.Network) (map[string]*netlink.Neigh, error) { + arps := make(map[string]*netlink.Neigh) + for k, v := range network.LocalNodeInfo { + if vx.nodeName == k { + continue + } + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(v.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error()) + } + nh := &netlink.Neigh{ LinkIndex: vx.vxlanIface.Attrs().Index, - State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, + State: netlink.NUD_PERMANENT, Type: netlink.NDA_DST, - Family: syscall.AF_BRIDGE, + Family: syscall.AF_INET, Flags: netlink.NTF_SELF, - IP: net.ParseIP(network.LocalEndpoint.PrivateIP), + IP: vxlanIP(net.ParseIP(v.PrivateIP)), HardwareAddr: HardwareAddr, - }, - }, nil + } + arps[networkutil.NeighKey(nh)] = nh + } + return arps, nil +} + +// calARPOnNonGateway calculates and returns the desired ARP entries on non-gateway node. +// The ARP entries format are equivalent to the following `ip neigh` command: +func (vx *vxlan) calARPOnNonGateway(network *types.Network) (map[string]*netlink.Neigh, error) { + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(network.LocalEndpoint.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error()) + } + nh := &netlink.Neigh{ + LinkIndex: vx.vxlanIface.Attrs().Index, + State: netlink.NUD_PERMANENT, + Type: netlink.NDA_DST, + Family: syscall.AF_INET, + Flags: netlink.NTF_SELF, + IP: vxlanIP(net.ParseIP(network.LocalEndpoint.PrivateIP)), + HardwareAddr: HardwareAddr, + } + return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil } // calIPSetOnNonGateway calculates and returns the desired ip set entries on non-gateway node. diff --git a/pkg/networkengine/util/netlink/netlink.go b/pkg/networkengine/util/netlink/netlink.go index 050e901..2c05366 100644 --- a/pkg/networkengine/util/netlink/netlink.go +++ b/pkg/networkengine/util/netlink/netlink.go @@ -42,9 +42,10 @@ var ( XfrmPolicyFlush = xfrmPolicyFlush - NeighAppend = neighAppend - NeighList = neighList - NeighDel = neighDel + NeighAdd = neighAdd + NeighReplace = neighReplace + NeighList = neighList + NeighDel = neighDel LinkByName = linkByName LinkByIndex = linkByIndex @@ -158,13 +159,23 @@ func ruleDel(rule *netlink.Rule) (err error) { return } -func neighAppend(neigh *netlink.Neigh) (err error) { - err = netlink.NeighAppend(neigh) +func neighAdd(neigh *netlink.Neigh) (err error) { + err = netlink.NeighAdd(neigh) if err != nil { - klog.ErrorS(err, "error on netlink.NeighAppend") + klog.ErrorS(err, "error on netlink.NeighSet") return } - klog.V(5).InfoS("netlink.NeighAppend succeeded") + klog.V(5).InfoS("netlink.NeighAdd succeeded") + return +} + +func neighReplace(neigh *netlink.Neigh) (err error) { + err = netlink.NeighSet(neigh) + if err != nil { + klog.ErrorS(err, "error on netlink.NeighSet") + return + } + klog.V(5).InfoS("netlink.NeighSet succeeded") return } diff --git a/pkg/networkengine/util/utils.go b/pkg/networkengine/util/utils.go index a2f6f7f..23b0add 100644 --- a/pkg/networkengine/util/utils.go +++ b/pkg/networkengine/util/utils.go @@ -49,6 +49,8 @@ func RouteKey(route *netlink.Route) string { return route.String() } +func NeighKey(neigh *netlink.Neigh) string { return neigh.String() } + func RuleKey(rule *netlink.Rule) string { return rule.String() } @@ -102,22 +104,22 @@ func ApplyRules(current, desired map[string]*netlink.Rule) (err error) { klog.InfoS("applying rules", "current", current, "desired", desired) } errList := errorlist.List{} - for k, v := range desired { - _, ok := current[k] + // deleted unwanted ip rules + for k, v := range current { + _, ok := desired[k] if !ok { - klog.InfoS("adding rule", "src", v.Src, "lookup", v.Table) - err = netlinkutil.RuleAdd(v) + klog.InfoS("deleting rule", "src", v.Src, "lookup", v.Table) + err = netlinkutil.RuleDel(v) errList = errList.Append(err) - continue } - delete(current, k) } - // remove unwanted rules - for _, v := range current { - klog.InfoS("deleting rule", "src", v.Src, "lookup", v.Table) - err = netlinkutil.RuleDel(v) + // add expect ip rules + for _, v := range desired { + klog.InfoS("adding rule", "src", v.Src, "lookup", v.Table) + err = netlinkutil.RuleAdd(v) errList = errList.Append(err) } + return errList.AsError() } @@ -126,25 +128,27 @@ func ApplyRoutes(current, desired map[string]*netlink.Route) (err error) { klog.InfoS("applying routes", "current", current, "desired", desired) } errList := errorlist.List{} - for k, v := range desired { - ro, ok := current[k] + for k, v := range current { + ro, ok := desired[k] if !ok { - klog.InfoS("adding route", "dst", v.Dst, "via", v.Gw, "src", v.Src, "table", v.Table) - err = netlinkutil.RouteAdd(v) - errList = errList.Append(err) - continue - } - delete(current, k) - if !ro.Equal(*v) { - klog.InfoS("replacing route", "dst", v.Dst, "via", v.Gw, "src", v.Src, "table", v.Table) - err = netlinkutil.RouteReplace(v) + // remove unwanted routes + klog.InfoS("deleting route", "dst", v.Dst.String(), "via", v.Gw.String()) + err = netlinkutil.RouteDel(v) errList = errList.Append(err) + } else { + // replace unequal routes + if !ro.Equal(*v) { + klog.InfoS("replacing route", "dst", v.Dst, "via", v.Gw, "src", v.Src, "table", v.Table) + err = netlinkutil.RouteReplace(v) + errList = errList.Append(err) + } + delete(desired, k) } } - // remove unwanted routes - for _, v := range current { - klog.InfoS("deleting route", "dst", v.Dst.String(), "via", v.Gw.String()) - err = netlinkutil.RouteDel(v) + // add new routes + for _, v := range desired { + klog.InfoS("adding route", "dst", v.Dst, "via", v.Gw, "src", v.Src, "table", v.Table) + err = netlinkutil.RouteAdd(v) errList = errList.Append(err) } return errList.AsError() @@ -155,20 +159,19 @@ func ApplyIPSet(set ipsetutil.IPSetInterface, current, desired map[string]*netli klog.InfoS("applying ipset entry", "current", current, "desired", desired) } errList := errorlist.List{} - for k, v := range desired { - _, ok := current[k] + // delete unwanted ipset entries + for k, v := range current { + _, ok := desired[k] if !ok { - klog.InfoS("adding entry", "entry", k) - err = set.Add(v) + klog.InfoS("deleting ipset entry", "entry", k) + err = set.Del(v) errList = errList.Append(err) - continue } - delete(current, k) } - // remove unwanted entries - for k, v := range current { - klog.InfoS("deleting ipset entry", "entry", k) - err = set.Del(v) + // add wanted ipset entries + for k, v := range desired { + klog.InfoS("adding entry", "entry", k) + err = set.Add(v) errList = errList.Append(err) } return errList.AsError() @@ -181,34 +184,76 @@ func ListFDBsOnNode(link netlink.Link) (map[string]*netlink.Neigh, error) { return nil, err } for k, v := range neighs { - if v.HardwareAddr.String() == AllZeroMAC.String() { - fdbsOnNode[v.IP.String()] = &neighs[k] - } + fdbsOnNode[v.String()] = &neighs[k] } return fdbsOnNode, nil } +func ListARPsOnNode(link netlink.Link) (map[string]*netlink.Neigh, error) { + arpsOnNode := make(map[string]*netlink.Neigh) + neighs, err := netlinkutil.NeighList(link.Attrs().Index, syscall.AF_INET) + if err != nil { + return nil, err + } + for k, v := range neighs { + arpsOnNode[v.String()] = &neighs[k] + } + return arpsOnNode, nil +} + func ApplyFDBs(current, desired map[string]*netlink.Neigh) (err error) { if klog.V(5).Enabled() { klog.InfoS("applying FDBs", "current", current, "desired", desired) } errList := errorlist.List{} - for k, v := range desired { - _, ok := current[k] + for k, v := range current { + ipNeigh, ok := desired[k] if !ok { - klog.InfoS("adding FDB", "dst", v.IP, "mac", v.HardwareAddr) - err = netlinkutil.NeighAppend(v) + klog.InfoS("deleting FDB", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighDel(v) + errList = errList.Append(err) + } else { + klog.InfoS("replace FDB", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighReplace(ipNeigh) errList = errList.Append(err) - continue } - delete(current, k) + delete(desired, k) } - // remove unwanted fdb entries - for _, v := range current { - klog.InfoS("deleting FDB", "dst", v.IP, "mac", v.HardwareAddr) - err = netlinkutil.NeighDel(v) + + for _, v := range desired { + klog.InfoS("adding FDB", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighAdd(v) errList = errList.Append(err) } + + return errList.AsError() +} + +func ApplyARPs(current, desired map[string]*netlink.Neigh) (err error) { + if klog.V(5).Enabled() { + klog.InfoS("applying ARPs", "current", current, "desired", desired) + } + errList := errorlist.List{} + for k, v := range current { + ipNeigh, ok := desired[k] + if !ok { + klog.InfoS("deleting ARP", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighDel(v) + errList = errList.Append(err) + } else { + klog.InfoS("replace ARP", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighReplace(ipNeigh) + errList = errList.Append(err) + } + delete(desired, k) + } + + for _, v := range desired { + klog.InfoS("adding ARP", "dst", v.IP, "mac", v.HardwareAddr) + err = netlinkutil.NeighAdd(v) + errList = errList.Append(err) + } + return errList.AsError() }