From d70eb1694cab96b8a41f316eff8fc3a9f51345bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=A9=E8=BD=A9?= Date: Thu, 21 Mar 2024 18:51:09 +0800 Subject: [PATCH] fix mac address for vxlan --- cmd/agent/app/config/config.go | 1 + cmd/agent/app/options/options.go | 18 ++++++ pkg/networkengine/routedriver/vxlan/vxlan.go | 59 +++++++++++++++---- .../routedriver/vxlan/vxlan_test.go | 3 +- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/cmd/agent/app/config/config.go b/cmd/agent/app/config/config.go index d5310f3..b9b4a3b 100644 --- a/cmd/agent/app/config/config.go +++ b/cmd/agent/app/config/config.go @@ -38,6 +38,7 @@ type TunnelConfig struct { VPNDriver string VPNPort string RouteDriver string + MACPrefix string ForwardNodeIP bool NATTraversal bool KeepAliveInterval int diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 3b09d91..963ddd6 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -5,7 +5,9 @@ import ( "fmt" "net" "os" + "regexp" "strconv" + "strings" "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" @@ -35,6 +37,7 @@ const ( DefaultTunnelMetricsPort = 10265 DefaultProxyMetricsPort = 10266 DefaultHealthyProbeAddr = 10275 + DefaultMACPrefix = "aa:0f" ) // AgentOptions has the information that required by the raven agent @@ -52,6 +55,7 @@ type TunnelOptions struct { VPNDriver string VPNPort string RouteDriver string + MACPrefix string ForwardNodeIP bool NATTraversal bool KeepAliveInterval int @@ -77,6 +81,15 @@ func (o *AgentOptions) Validate() error { return errors.New("currently only supports libreswan and wireguard VPN drivers") } } + if o.MACPrefix != "" { + reg := regexp.MustCompile(`^[0-9a-fA-F]+$`) + strs := strings.Split(o.MACPrefix, ":") + for i := range strs { + if !reg.MatchString(strings.ToLower(strs[i])) { + return fmt.Errorf("mac prefix %s is nonstandard", o.MACPrefix) + } + } + } return nil } @@ -95,6 +108,7 @@ func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.ForwardNodeIP, "forward-node-ip", o.ForwardNodeIP, `Forward node IP or not. (default "false")`) fs.IntVar(&o.KeepAliveInterval, "keep-alive-interval", o.KeepAliveInterval, `Interval for sending keepalive packets in the VPN tunnel, (default "0", closed)`) fs.IntVar(&o.KeepAliveTimeout, "keep-alive-timeout", o.KeepAliveTimeout, `Timeout for sending keepalive packets in the VPN tunnel, (default "0", closed)`) + fs.StringVar(&o.MACPrefix, "customized-mac-prefix", o.MACPrefix, `Customized MAC address prefix for vxlan link, (default "aa:0f")`) fs.StringVar(&o.ProxyMetricsAddress, "proxy-metric-bind-addr", o.ProxyMetricsAddress, `Binding address of proxy metrics. (default ":10266")`) fs.StringVar(&o.InternalSecureAddress, "proxy-internal-secure-addr", o.InternalSecureAddress, `Binding secure address of proxy server. (default ":10263")`) @@ -145,6 +159,7 @@ func (o *AgentOptions) Config() (*config.Config, error) { VPNPort: port, VPNDriver: o.VPNDriver, RouteDriver: o.RouteDriver, + MACPrefix: o.MACPrefix, ForwardNodeIP: o.ForwardNodeIP, NATTraversal: o.NATTraversal, KeepAliveInterval: o.KeepAliveInterval, @@ -171,6 +186,9 @@ func (o *AgentOptions) Config() (*config.Config, error) { if c.Tunnel.VPNPort == "" { c.Tunnel.VPNPort = vpndriver.DefaultVPNPort } + if c.Tunnel.MACPrefix == "" { + c.Tunnel.MACPrefix = DefaultMACPrefix + } if c.Proxy.ProxyClientCertDir == "" { c.Proxy.ProxyClientCertDir = utils.RavenProxyClientCertDir } diff --git a/pkg/networkengine/routedriver/vxlan/vxlan.go b/pkg/networkengine/routedriver/vxlan/vxlan.go index 2390c97..3bd7601 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan.go @@ -22,6 +22,7 @@ import ( "math" "net" "os" + "strings" "syscall" "github.com/vdobler/ht/errorlist" @@ -67,6 +68,7 @@ func init() { type vxlan struct { vxlanIface netlink.Link nodeName types.NodeName + macPrefix string iptables iptablesutil.IPTablesInterface ipset ipsetutil.IPSetInterface @@ -137,7 +139,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error if vx.isGatewayRole(network) { desiredRoutes = vx.calRouteOnGateway(network) - desiredFDBs = vx.calFDBOnGateway(network) + desiredFDBs, err = vx.calFDBOnGateway(network) + if err != nil { + return fmt.Errorf("error calculate gateway fdb: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec) if err != nil { @@ -149,8 +154,10 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error } } else { desiredRoutes = vx.calRouteOnNonGateway(network) - desiredFDBs = vx.calFDBOnNonGateway(network) - + desiredFDBs, err = vx.calFDBOnNonGateway(network) + if err != nil { + return fmt.Errorf("error calculate non gateway fdb: %s", err) + } err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec) if err != nil { return fmt.Errorf("error deleting gateway chain rule: %s", err) @@ -218,7 +225,8 @@ func (vx *vxlan) nodeInfo(network *types.Network) *v1beta1.NodeInfo { func New(cfg *config.Config) (routedriver.Driver, error) { return &vxlan{ - nodeName: types.NodeName(cfg.NodeName), + nodeName: types.NodeName(cfg.NodeName), + macPrefix: cfg.Tunnel.MACPrefix, }, nil } @@ -274,13 +282,17 @@ func (vx *vxlan) ensureVxlanLink(network *types.Network, vpnDriverMTUFn func() ( VxlanId: vxlanID, Age: 300, Port: vxlanPort, - Learning: true, + Learning: false, } if !vx.isGatewayRole(network) { vxlanLink.Group = net.ParseIP(network.LocalEndpoint.PrivateIP) } localIP := net.ParseIP(vx.nodeInfo(network).PrivateIP) + vxlanLink.HardwareAddr, err = vx.ipAddrToHardwareAddr(localIP) + if err != nil { + return fmt.Errorf("error convert vxlan ip to mac address: %s", err) + } nl, err := ensureVxlanLink(vxlanLink, vxlanIP(localIP)) if err != nil { return fmt.Errorf("error ensuring vxlan link: %s", err) @@ -383,13 +395,17 @@ func (vx *vxlan) calRulesOnNode() map[string]*netlink.Rule { // calFDBOnGateway calculates and returns the desired FDB entries on gateway node. // The FDB entries format are equivalent to the following `bridge fdb append` command: // -// bridge fdb append 00:00:00:00:00:00 dev raven0 dst {non_gateway_nodeN_private_ip} self permanent -func (vx *vxlan) calFDBOnGateway(network *types.Network) map[string]*netlink.Neigh { +// bridge fdb append fixed mac address dev raven0 dst {non_gateway_nodeN_private_ip} self permanent +func (vx *vxlan) calFDBOnGateway(network *types.Network) (map[string]*netlink.Neigh, error) { fdbs := make(map[string]*netlink.Neigh) for k, v := range network.LocalNodeInfo { if vx.nodeName == k { continue } + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(v.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", v.PrivateIP, err.Error()) + } fdbs[v.PrivateIP] = &netlink.Neigh{ LinkIndex: vx.vxlanIface.Attrs().Index, State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, @@ -397,17 +413,21 @@ func (vx *vxlan) calFDBOnGateway(network *types.Network) map[string]*netlink.Nei Family: syscall.AF_BRIDGE, Flags: netlink.NTF_SELF, IP: net.ParseIP(v.PrivateIP), - HardwareAddr: networkutil.AllZeroMAC, + HardwareAddr: HardwareAddr, } } - return fdbs + return fdbs, nil } // calFDBOnNonGateway calculates and returns the desired FDB entries on non-gateway node. // The FDB entries format are equivalent to the following `bridge fdb append` command: // -// bridge fdb append 00:00:00:00:00:00 dev raven0 dst {gateway_node_private_ip} self permanent -func (vx *vxlan) calFDBOnNonGateway(network *types.Network) map[string]*netlink.Neigh { +// bridge fdb append fixed mac address dev raven0 dst {gateway_node_private_ip} self permanent +func (vx *vxlan) calFDBOnNonGateway(network *types.Network) (map[string]*netlink.Neigh, error) { + HardwareAddr, err := vx.ipAddrToHardwareAddr(net.ParseIP(network.LocalEndpoint.PrivateIP)) + if err != nil { + return nil, fmt.Errorf("convert ip address %s to hardware address error %s", network.LocalEndpoint.PrivateIP, err.Error()) + } return map[string]*netlink.Neigh{ network.LocalEndpoint.PrivateIP: { LinkIndex: vx.vxlanIface.Attrs().Index, @@ -416,9 +436,9 @@ func (vx *vxlan) calFDBOnNonGateway(network *types.Network) map[string]*netlink. Family: syscall.AF_BRIDGE, Flags: netlink.NTF_SELF, IP: net.ParseIP(network.LocalEndpoint.PrivateIP), - HardwareAddr: networkutil.AllZeroMAC, + HardwareAddr: HardwareAddr, }, - } + }, nil } // calIPSetOnNonGateway calculates and returns the desired ip set entries on non-gateway node. @@ -522,3 +542,16 @@ func (vx *vxlan) isGatewayRole(network *types.Network) bool { network.LocalEndpoint != nil && network.LocalEndpoint.NodeName == vx.nodeName } + +func (vx *vxlan) ipAddrToHardwareAddr(ip net.IP) (net.HardwareAddr, error) { + macSlice := []string{vx.macPrefix} + for _, ipSlice := range vxlanIP(ip).To4() { + macSlice = append(macSlice, fmt.Sprintf("%02x", ipSlice)) + } + macStr := strings.Join(macSlice, ":") + macAddr, err := net.ParseMAC(macStr) + if err != nil { + return nil, fmt.Errorf("parse MAC %s error %s", macStr, err.Error()) + } + return macAddr, nil +} diff --git a/pkg/networkengine/routedriver/vxlan/vxlan_test.go b/pkg/networkengine/routedriver/vxlan/vxlan_test.go index 018b0a8..c529e66 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan_test.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan_test.go @@ -440,7 +440,8 @@ func TestVxlan_Apply(t *testing.T) { for _, v := range testcases { t.Run(v.name, func(t *testing.T) { vx := vxlan{ - nodeName: v.nodeName, + nodeName: v.nodeName, + macPrefix: "aa:0f", } a := assert.New(t) a.NoError(vx.Init())