1. fix the bug of raven l7 is closed due to gateway is deleted

2. fix the bug of raven l3 can not config arp

cmd/agent/app/options/options.go

@@ -37,6 +37,7 @@ const (
 	DefaultTunnelMetricsPort = 10265
 	DefaultProxyMetricsPort  = 10266
 	DefaultHealthyProbeAddr  = 10275
+	DefaultLocalHost         = "127.0.0.1"
 	DefaultMACPrefix         = "aa:0f"
 )
 
@@ -144,7 +145,7 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 		NodeIP:   o.NodeIP,
 	}
 	c.KubeConfig = cfg
-	c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, c.NodeIP, strconv.Itoa(DefaultTunnelMetricsPort))
+	c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, resolveLocalHost(), strconv.Itoa(DefaultTunnelMetricsPort))
 	c.HealthProbeAddr = resolveAddress(c.HealthProbeAddr, c.NodeIP, strconv.Itoa(DefaultHealthyProbeAddr))
 	c.Manager, err = newMgr(cfg, c.MetricsBindAddress, c.HealthProbeAddr)
 	if err != nil {
@@ -202,7 +203,7 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 	c.Proxy.InternalInsecureAddress = resolveAddress(c.Proxy.InternalInsecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerInsecurePort))
 	c.Proxy.InternalSecureAddress = resolveAddress(c.Proxy.InternalSecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerSecurePort))
 	c.Proxy.ExternalAddress = resolveAddress(c.Proxy.ExternalAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerExposedPort))
-	c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, c.NodeIP, strconv.Itoa(DefaultProxyMetricsPort))
+	c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, resolveLocalHost(), strconv.Itoa(DefaultProxyMetricsPort))
 
 	return c, nil
 }
@@ -310,6 +311,15 @@ func getGatewayAPIGroupResource() *restmapper.APIGroupResources {
 	}
 }
 
+func resolveLocalHost() string {
+	ipv4Addr, err := net.ResolveIPAddr("ip4", "localhost")
+	if err != nil {
+		klog.Warningf("can not get localhost addr, error %s, using default address %s", err.Error(), DefaultLocalHost)
+		return DefaultLocalHost
+	}
+	return ipv4Addr.String()
+}
+
 func resolveAddress(srcAddr, defaultHost, defaultPort string) string {
 	if srcAddr == "" {
 		return net.JoinHostPort(defaultHost, defaultPort)

pkg/engine/proxy.go

@@ -97,16 +97,22 @@ func (p *ProxyEngine) processNextWorkItem() bool {
 }
 
 func (p *ProxyEngine) handler(gw *v1beta1.Gateway) error {
-	proxyStatus := enableProxy(gw)
-	p.option.SetProxyStatus(proxyStatus)
-	specServer, specClient := p.getRole(proxyStatus)
 	var err error
 	p.gateway, err = utils.GetOwnGateway(p.client, p.nodeName)
 	if err != nil {
 		klog.Errorf(utils.FormatProxyServer("failed get gateway for %s, can not start proxy server", p.nodeName))
 		return err
 	}
-
+	proxyStatus := p.option.GetProxyStatus()
+	if p.gateway != nil && gw.GetName() == p.gateway.GetName() {
+		proxyStatus = enableProxy(gw)
+	} else {
+		if gw.Spec.ExposeType != "" {
+			proxyStatus = enableProxy(gw)
+		}
+	}
+	p.option.SetProxyStatus(proxyStatus)
+	specServer, specClient := p.getRole(proxyStatus)
 	switch JudgeType(p.proxyOption.GetServerStatus(), specServer) {
 	case StartType:
 		srcAddr := getSrcAddressForProxyServer(p.client, p.nodeName)

pkg/networkengine/routedriver/vxlan/vxlan.go

@@ -93,9 +93,13 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
 	var desiredRules, currentRules map[string]*netlink.Rule
 
 	// The desired and current FDB entries calculated from given network.
-	// The key is netlink.Neigh.IP
+	// The key is NeighKey()
 	var desiredFDBs, currentFDBs map[string]*netlink.Neigh
 
+	// The desired and current ARP entries calculated from given network.
+	// The key is NeighKey()
+	var desiredARPs, currentARPs map[string]*netlink.Neigh
+
 	// The desired and current ipset entries calculated from given network.
 	// The key is ip set entry
 	var desiredSet, currentSet map[string]*netlink.IPSetEntry
@@ -129,6 +133,11 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
 		return fmt.Errorf("error listing fdb on node: %s", err)
 	}
 
+	currentARPs, err = networkutil.ListARPsOnNode(vx.vxlanIface)
+	if err != nil {
+		return fmt.Errorf("error listing arp on node: %s", err)
+	}
+
 	currentSet, err = networkutil.ListIPSetOnNode(vx.ipset)
 	if err != nil {
 		return fmt.Errorf("error listing ip set on node: %s", err)
@@ -143,7 +152,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
 		if err != nil {
 			return fmt.Errorf("error calculate gateway fdb: %s", err)
 		}
-
+		desiredARPs, err = vx.calARPOnGateway(network)
+		if err != nil {
+			return fmt.Errorf("error calculate gateway arp: %s", err)
+		}
 		err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec)
 		if err != nil {
 			return fmt.Errorf("error deleting non gateway chain rule: %s", err)
@@ -158,6 +170,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
 		if err != nil {
 			return fmt.Errorf("error calculate non gateway fdb: %s", err)
 		}
+		desiredARPs, err = vx.calARPOnNonGateway(network)
+		if err != nil {
+			return fmt.Errorf("error calculate non gateway arp: %s", err)
+		}
 		err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec)
 		if err != nil {
 			return fmt.Errorf("error deleting gateway chain rule: %s", err)
@@ -180,6 +196,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
 	if err != nil {
 		return fmt.Errorf("error applying fdb: %s", err)
 	}
+	err = networkutil.ApplyARPs(currentARPs, desiredARPs)
+	if err != nil {
+		return fmt.Errorf("error applying arp: %s", err)
+	}
 	err = networkutil.ApplyIPSet(vx.ipset, currentSet, desiredSet)
 	if err != nil {
 		return fmt.Errorf("error applying ip set: %s", err)
@@ -406,7 +426,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne
 		if err != nil {
 			return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error())
 		}
-		fdbs[v.PrivateIP] = &netlink.Neigh{
+		nh := &netlink.Neigh{
 			LinkIndex:    vx.vxlanIface.Attrs().Index,
 			State:        netlink.NUD_PERMANENT | netlink.NUD_NOARP,
 			Type:         netlink.NDA_DST,
@@ -415,6 +435,7 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Ne
 			IP:           net.ParseIP(v.PrivateIP),
 			HardwareAddr: HardwareAddr,
 		}
+		fdbs[networkutil.NeighKey(nh)] = nh
 	}
 	return fdbs, nil
 }
@@ -428,17 +449,62 @@ func (vx *vxlan) calFDBOnNonGateway(network *types.Network) (map[string]*netlink
 	if err != nil {
 		return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error())
 	}
-	return map[string]*netlink.Neigh{
-		network.LocalEndpoint.PrivateIP: {
+	nh := &netlink.Neigh{
+		LinkIndex:    vx.vxlanIface.Attrs().Index,
+		State:        netlink.NUD_PERMANENT | netlink.NUD_NOARP,
+		Type:         netlink.NDA_DST,
+		Family:       syscall.AF_BRIDGE,
+		Flags:        netlink.NTF_SELF,
+		IP:           net.ParseIP(network.LocalEndpoint.PrivateIP),
+		HardwareAddr: HardwareAddr,
+	}
+	return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil
+}
+
+// calARPOnGateway calculates and returns the desired ARP entries on gateway node.
+// The ARP entries format are equivalent to the following `ip neigh` command:
+func (vx *vxlan) calARPOnGateway(network *types.Network) (map[string]*netlink.Neigh, error) {
+	arps := make(map[string]*netlink.Neigh)
+	for k, v := range network.LocalNodeInfo {
+		if vx.nodeName == k {
+			continue
+		}
+		HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(v.PrivateIP))
+		if err != nil {
+			return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error())
+		}
+		nh := &netlink.Neigh{
 			LinkIndex:    vx.vxlanIface.Attrs().Index,
-			State:        netlink.NUD_PERMANENT | netlink.NUD_NOARP,
+			State:        netlink.NUD_PERMANENT,
 			Type:         netlink.NDA_DST,
-			Family:       syscall.AF_BRIDGE,
+			Family:       syscall.AF_INET,
 			Flags:        netlink.NTF_SELF,
-			IP:           net.ParseIP(network.LocalEndpoint.PrivateIP),
+			IP:           vxlanIP(net.ParseIP(v.PrivateIP)),
 			HardwareAddr: HardwareAddr,
-		},
-	}, nil
+		}
+		arps[networkutil.NeighKey(nh)] = nh
+	}
+	return arps, nil
+}
+
+// calARPOnNonGateway calculates and returns the desired ARP entries on non-gateway node.
+// The ARP entries format are equivalent to the following `ip neigh` command:
+//
+func (vx *vxlan) calARPOnNonGateway(network *types.Network) (map[string]*netlink.Neigh, error) {
+	HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(network.LocalEndpoint.PrivateIP))
+	if err != nil {
+		return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error())
+	}
+	nh := &netlink.Neigh{
+		LinkIndex:    vx.vxlanIface.Attrs().Index,
+		State:        netlink.NUD_PERMANENT,
+		Type:         netlink.NDA_DST,
+		Family:       syscall.AF_INET,
+		Flags:        netlink.NTF_SELF,
+		IP:           vxlanIP(net.ParseIP(network.LocalEndpoint.PrivateIP)),
+		HardwareAddr: HardwareAddr,
+	}
+	return map[string]*netlink.Neigh{networkutil.NeighKey(nh): nh}, nil
 }
 
 // calIPSetOnNonGateway calculates and returns the desired ip set entries on non-gateway node.

pkg/networkengine/util/netlink/netlink.go

@@ -42,9 +42,10 @@ var (
 
 	XfrmPolicyFlush = xfrmPolicyFlush
 
-	NeighAppend = neighAppend
-	NeighList   = neighList
-	NeighDel    = neighDel
+	NeighAdd     = neighAdd
+	NeighReplace = neighReplace
+	NeighList    = neighList
+	NeighDel     = neighDel
 
 	LinkByName  = linkByName
 	LinkByIndex = linkByIndex
@@ -158,13 +159,23 @@ func ruleDel(rule *netlink.Rule) (err error) {
 	return
 }
 
-func neighAppend(neigh *netlink.Neigh) (err error) {
-	err = netlink.NeighAppend(neigh)
+func neighAdd(neigh *netlink.Neigh) (err error) {
+	err = netlink.NeighAdd(neigh)
 	if err != nil {
-		klog.ErrorS(err, "error on netlink.NeighAppend")
+		klog.ErrorS(err, "error on netlink.NeighSet")
 		return
 	}
-	klog.V(5).InfoS("netlink.NeighAppend succeeded")
+	klog.V(5).InfoS("netlink.NeighAdd succeeded")
+	return
+}
+
+func neighReplace(neigh *netlink.Neigh) (err error) {
+	err = netlink.NeighSet(neigh)
+	if err != nil {
+		klog.ErrorS(err, "error on netlink.NeighSet")
+		return
+	}
+	klog.V(5).InfoS("netlink.NeighSet succeeded")
 	return
 }