feat: support autoconfig Raven gateway topology across multi-edge net…

…work

Signed-off-by: Liang Deng <[email protected]>

Makefile

@@ -2,7 +2,8 @@
 # Image URL to use all building/pushing image targets
 IMG ?= openyurt/raven-agent:latest
 VPN_DRIVER ?= libreswan
-FORWARD_NODE_IP ?= false
+FORWARD_NODE_IP ?= true
+NAT_TRAVERSAL ?= false
 METRIC_BIND_ADDR ?= ":8080"
 
 BUILDPLATFORM ?= linux/amd64
@@ -65,7 +66,7 @@ docker-push: ## Push docker image with the agent.
 ##@ Deploy
 
 gen-deploy-yaml:
-	bash hack/gen-yaml.sh ${IMG} ${VPN_DRIVER} ${FORWARD_NODE_IP} ${METRIC_BIND_ADDR}
+	bash hack/gen-yaml.sh ${IMG} ${VPN_DRIVER} ${FORWARD_NODE_IP} ${METRIC_BIND_ADDR} ${NAT_TRAVERSAL}
 
 deploy: gen-deploy-yaml ## Deploy agent daemon.
 	kubectl apply -f _output/yamls/raven-agent.yaml

charts/raven-agent/templates/config.yaml

@@ -11,6 +11,7 @@ apiVersion: v1
 data:
   vpn-driver: {{ .Values.vpn.driver }}
   forward-node-ip: {{ .Values.vpn.forwardNodeIP | quote }}
+  nat-traversal: {{ .Values.vpn.natTraversal | quote }}
   metric-bind-addr: {{ .Values.vpn.metricBindAddr }}
   tunnel-bind-addr: {{ .Values.vpn.tunnelAddr }}
   proxy-external-addr: {{ .Values.proxy.externalAddr }}

charts/raven-agent/templates/daemonset.yaml

@@ -30,6 +30,7 @@ spec:
           - --v=2
           - --vpn-driver={{.Values.vpn.driver}}
           - --forward-node-ip={{.Values.vpn.forwardNodeIP}}
+          - --nat-traversal={{.Values.vpn.natTraversal}}
           - --metric-bind-addr={{.Values.vpn.metricBindAddr}}
           - --vpn-bind-port={{.Values.vpn.tunnelAddr}}
           - --proxy-metric-bind-addr={{.Values.proxy.metricsBindAddr}}

charts/raven-agent/values.yaml

@@ -62,6 +62,7 @@ containerEnv:
 vpn:
   driver: libreswan
   forwardNodeIP: true
+  natTraversal: false
   # raven-agent requires a unique vpn psk
   # You can generate one with the command:
   # 'openssl rand -hex 64'

cmd/agent/app/config/config.go

@@ -37,6 +37,7 @@ type TunnelConfig struct {
 	VPNPort       string
 	RouteDriver   string
 	ForwardNodeIP bool
+	NATTraversal  bool
 }
 
 type ProxyConfig struct {

cmd/agent/app/options/options.go

@@ -45,6 +45,7 @@ type TunnelOptions struct {
 	VPNPort       string
 	RouteDriver   string
 	ForwardNodeIP bool
+	NATTraversal  bool
 }
 
 type ProxyOptions struct {
@@ -83,6 +84,7 @@ func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.VPNDriver, "vpn-driver", o.VPNDriver, `The VPN driver name. (default "libreswan")`)
 	fs.StringVar(&o.RouteDriver, "route-driver", o.RouteDriver, `The Route driver name. (default "vxlan")`)
 	fs.BoolVar(&o.ForwardNodeIP, "forward-node-ip", o.ForwardNodeIP, `Forward node IP or not. (default "false")`)
+	fs.BoolVar(&o.NATTraversal, "nat-traversal", o.NATTraversal, `Enable NAT Traversal or not. (default "false")`)
 	fs.StringVar(&o.MetricsBindAddress, "metric-bind-addr", o.MetricsBindAddress, `Binding address of tunnel metrics. (default ":10265")`)
 	fs.StringVar(&o.VPNPort, "vpn-bind-port", o.VPNPort, `Binding port of vpn. (default ":4500")`)
 	fs.StringVar(&o.ProxyMetricsAddress, "proxy-metric-bind-addr", o.ProxyMetricsAddress, `Binding address of proxy metrics. (default ":10266")`)
@@ -124,6 +126,7 @@ func (o *AgentOptions) Config() (*config.Config, error) {
 		VPNDriver:     o.VPNDriver,
 		RouteDriver:   o.RouteDriver,
 		ForwardNodeIP: o.ForwardNodeIP,
+		NATTraversal:  o.NATTraversal,
 	}
 	c.Proxy = &config.ProxyConfig{
 		ProxyMetricsAddress:     o.ProxyMetricsAddress,

config/raven-agent/agent/agent.yaml

@@ -43,6 +43,11 @@ spec:
                 configMapKeyRef:
                   name: agent-config
                   key: forward-node-ip
+            - name: NAT_TRAVERSAL
+              valueFrom:
+                configMapKeyRef:
+                  name: agent-config
+                  key: nat-traversal
             - name: METRIC_BIND_ADDR
               valueFrom:
                 configMapKeyRef:

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a
+	github.com/ccding/go-stun v0.1.5-0.20230908213042-0f417a9a4966
 	github.com/coreos/go-iptables v0.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/lorenzosaino/go-sysctl v0.3.1

go.sum

@@ -96,6 +96,8 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/ccding/go-stun v0.1.5-0.20230908213042-0f417a9a4966 h1:ugTbop8ITMmnyZRFFQZ0LDnEi+m28dDU7Jxf6cYoA5M=
+github.com/ccding/go-stun v0.1.5-0.20230908213042-0f417a9a4966/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=

hack/gen-yaml.sh

@@ -22,6 +22,7 @@ gen_yaml() {
     local VPN_DRIVER=$2
     local FORWARD_NODE_IP=$3
     local METRIC_BIND_ADDR=$4
+    local NAT_TRAVERSAL=$5
     local OUT_YAML_DIR=${YURT_ROOT}/_output/yamls
     local BUILD_YAML_DIR=${OUT_YAML_DIR}/build
     [ -f "${BUILD_YAML_DIR}" ] || mkdir -p "${BUILD_YAML_DIR}"
@@ -38,6 +39,7 @@ gen_yaml() {
     [ -f "${BUILD_YAML_DIR}"/default/config.env ] || echo "vpn-driver=${VPN_DRIVER}" > "${BUILD_YAML_DIR}"/default/config.env
     echo "forward-node-ip=${FORWARD_NODE_IP}" >> "${BUILD_YAML_DIR}"/default/config.env
     echo "metric-bind-addr=${METRIC_BIND_ADDR}" >> "${BUILD_YAML_DIR}"/default/config.env
+    echo "nat-traversal=${NAT_TRAVERSAL}" >> "${BUILD_YAML_DIR}"/default/config.env
     kustomize build "${BUILD_YAML_DIR}"/default > "${OUT_YAML_DIR}"/raven-agent.yaml
     rm -Rf "${BUILD_YAML_DIR}"
 }

pkg/engine/tunnel.go

@@ -53,6 +53,9 @@ func (t *TunnelEngine) processNextWorkItem() bool {
 
 func (t *TunnelEngine) handler(gw *v1beta1.Gateway) error {
 	klog.Info(utils.FormatRavenEngine("update raven l3 tunnel config for gateway %s", gw.GetName()))
+	if err := t.getStunInfo(); err != nil {
+		return err
+	}
 	if t.routeDriver == nil || t.vpnDriver == nil {
 		err := t.initDriver()
 		if err != nil {
@@ -104,6 +107,24 @@ func (t *TunnelEngine) clearDriver() error {
 	return nil
 }
 
+func (t *TunnelEngine) getStunInfo() error {
+	natType, err := utils.GetNATType()
+	if err != nil {
+		return err
+	}
+
+	if natType == utils.NATSymmetric {
+		return nil
+	}
+
+	_, err = utils.GetPublicPort()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (t *TunnelEngine) handleEventErr(err error, event interface{}) {
 	if err == nil {
 		t.queue.Forget(event)

pkg/networkengine/vpndriver/driver.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openyurtio/raven/cmd/agent/app/config"
 	netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink"
 	"github.com/openyurtio/raven/pkg/types"
+	"github.com/openyurtio/raven/pkg/utils"
 )
 
 const (
@@ -65,6 +66,7 @@ type Factory func(cfg *config.Config) (Driver, error)
 var (
 	driversMutex sync.Mutex
 	drivers      = make(map[string]Factory)
+	natTraversal bool
 )
 
 func RegisterDriver(name string, factory Factory) {
@@ -78,6 +80,9 @@ func RegisterDriver(name string, factory Factory) {
 }
 
 func New(name string, cfg *config.Config) (Driver, error) {
+	if cfg.Tunnel != nil {
+		natTraversal = cfg.Tunnel.NATTraversal
+	}
 	driversMutex.Lock()
 	defer driversMutex.Unlock()
 	if _, found := drivers[name]; !found {
@@ -110,6 +115,22 @@ func FindCentralGwFn(network *types.Network) *types.Endpoint {
 	return central
 }
 
+// EnableCreateEdgeConnection determine whether VPN tunnels can be established between edges.
+func EnableCreateEdgeConnection(localEndpoint *types.Endpoint, remoteEndpoint *types.Endpoint) bool {
+	if !natTraversal {
+		return false
+	}
+	if localEndpoint.NATType == utils.NATUndefined || remoteEndpoint.NATType == utils.NATUndefined {
+		return false
+	}
+	if !localEndpoint.UnderNAT || !remoteEndpoint.UnderNAT {
+		return false
+	}
+	return !((localEndpoint.NATType == utils.NATSymmetric && remoteEndpoint.NATType == utils.NATSymmetric) ||
+		(localEndpoint.NATType == utils.NATSymmetric && remoteEndpoint.NATType == utils.NATPortRestricted) ||
+		(localEndpoint.NATType == utils.NATPortRestricted && remoteEndpoint.NATType == utils.NATSymmetric))
+}
+
 func DefaultMTU() (int, error) {
 	routes, err := netlinkutil.RouteListFiltered(
 		netlink.FAMILY_V4,