diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9cc4444..e239157 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -46,7 +46,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: v1.51.2 - args: --timeout 300s --skip-dirs-use-default -v -E goconst -E gofmt -E ineffassign -E goimports -E revive -E misspell -E vet -E deadcode + args: --timeout 300s --skip-dirs-use-default -v -E goconst -E gofmt -E ineffassign -E goimports -E revive -E misspell -E vet -E unused shellcheck: name: Shellcheck diff --git a/Dockerfile b/Dockerfile index 258c539..017e6ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,6 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && rc-update add ipsec COPY --from=builder /workspace/raven-agent-ds /usr/local/bin/ -COPY pluto raven.sh /usr/local/bin/ +COPY pluto /usr/local/bin/ -ENTRYPOINT raven.sh -ENTRYPOINT ["/usr/local/bin/raven-agent-ds"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/raven-agent-ds"] \ No newline at end of file diff --git a/charts/raven-agent/templates/config.yaml b/charts/raven-agent/templates/config.yaml index bbc579e..206bead 100644 --- a/charts/raven-agent/templates/config.yaml +++ b/charts/raven-agent/templates/config.yaml @@ -1,8 +1,21 @@ apiVersion: v1 +data: + enable-l3-tunnel: "false" + enable-l7-proxy: "true" +kind: ConfigMap +metadata: + name: raven-cfg + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 data: vpn-driver: {{ .Values.vpn.driver }} forward-node-ip: {{ .Values.vpn.forwardNodeIP | quote }} metric-bind-addr: {{ .Values.vpn.metricBindAddr }} + tunnel-bind-addr: {{ .Values.vpn.tunnelAddr }} + proxy-external-addr: {{ .Values.proxy.externalAddr }} + proxy-internal-insecure-addr: {{ .Values.proxy.internalInsecureAddr }} + proxy-internal-secure-addr: {{ .Values.proxy.internalSecureAddr }} kind: ConfigMap metadata: name: raven-agent-config diff --git a/charts/raven-agent/templates/daemonset.yaml b/charts/raven-agent/templates/daemonset.yaml index f23894a..a0d05a5 100644 --- a/charts/raven-agent/templates/daemonset.yaml +++ b/charts/raven-agent/templates/daemonset.yaml @@ -26,6 +26,8 @@ spec: env: {{- toYaml . | nindent 8 }} {{- end }} + args: + - --v=4 hostNetwork: true {{- with .Values.nodeSelector }} nodeSelector: diff --git a/charts/raven-agent/templates/rbac.yaml b/charts/raven-agent/templates/rbac.yaml index 8f72de2..7d5ce61 100644 --- a/charts/raven-agent/templates/rbac.yaml +++ b/charts/raven-agent/templates/rbac.yaml @@ -19,6 +19,26 @@ rules: - watch - patch - update + - apiGroups: + - "" + resources: + - configmaps + - services + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - create + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -31,4 +51,43 @@ roleRef: subjects: - kind: ServiceAccount name: raven-agent-account - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: raven-proxy-user +rules: + - apiGroups: + - "" + resources: + - nodes/stats + - nodes/metrics + - nodes/log + - nodes/spec + - nodes/proxy + verbs: + - create + - get + - list + - watch + - delete + - update + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: raven-proxy-user-binding +subjects: + - kind: User + name: tunnel-proxy-client + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: raven-proxy-user + apiGroup: rbac.authorization.k8s.io +--- \ No newline at end of file diff --git a/charts/raven-agent/values.yaml b/charts/raven-agent/values.yaml index df221b5..95e36d3 100644 --- a/charts/raven-agent/values.yaml +++ b/charts/raven-agent/values.yaml @@ -30,7 +30,6 @@ securityContext: privileged: true nodeSelector: - kubernetes.io/arch: amd64 kubernetes.io/os: linux tolerations: [{"operator": "Exists"}] @@ -50,6 +49,10 @@ containerEnv: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: VPN_CONNECTION_PSK valueFrom: secretKeyRef: @@ -70,6 +73,26 @@ containerEnv: configMapKeyRef: key: metric-bind-addr name: raven-agent-config + - name: VPN_BIND_ADDRESS + valueFrom: + configMapKeyRef: + key: tunnel-bind-addr + name: raven-agent-config + - name: PROXY_SERVER_INTERNAL_SECURE_ADDRESS + valueFrom: + configMapKeyRef: + key: proxy-internal-secure-addr + name: raven-agent-config + - name: PROXY_SERVER_INTERNAL_INSECURE_ADDRESS + valueFrom: + configMapKeyRef: + key: proxy-internal-insecure-addr + name: raven-agent-config + - name: PROXY_SERVER_EXTERNAL_ADDRESS + valueFrom: + configMapKeyRef: + key: proxy-external-addr + name: raven-agent-config vpn: driver: libreswan forwardNodeIP: false @@ -79,6 +102,13 @@ vpn: # Pass it to helm with '--set vpn.psk=`openssl rand -hex 64`' # IMPORTANT: You should NOT use the example psk for a production deployment! psk: OPENYURT-RAVEN-AGENT-VPN-PSK - metricBindAddr: ":8080" + metricBindAddr: ":10265" + tunnelAddr: ":4500" +proxy: + externalAddr: ":10262" + internalInsecureAddr: ":10264" + internalSecureAddr: ":10263" + + rollingUpdate: maxUnavailable: 5% \ No newline at end of file diff --git a/cmd/agent/app/config/config.go b/cmd/agent/app/config/config.go index 3feada3..f27cd3b 100644 --- a/cmd/agent/app/config/config.go +++ b/cmd/agent/app/config/config.go @@ -24,12 +24,32 @@ import ( // Config is the main context object for raven agent type Config struct { NodeName string - Kubeconfig *rest.Config - Manager manager.Manager - VPNDriver string - RouteDriver string - ForwardNodeIP bool + NodeIP string MetricsBindAddress string + KubeConfig *rest.Config + Manager manager.Manager + Tunnel *TunnelConfig + Proxy *ProxyConfig +} + +type TunnelConfig struct { + VPNDriver string + VPNPort string + RouteDriver string + ForwardNodeIP bool +} + +type ProxyConfig struct { + ProxyMetricsAddress string + ProxyClientCertDir string + + InternalInsecureAddress string + InternalSecureAddress string + ExternalAddress string + ProxyServerCertDNSNames string + ProxyServerCertIPs string + ProxyServerCertDir string + InterceptorServerUDSFile string } type completedConfig struct { diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index 280e9e6..05aa1f9 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -3,10 +3,14 @@ package options import ( "errors" "fmt" + + "net" "os" + "strconv" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" + "github.com/openyurtio/raven/pkg/utils" "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -14,21 +18,47 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" "github.com/openyurtio/raven/cmd/agent/app/config" "github.com/openyurtio/raven/pkg/networkengine/routedriver/vxlan" + "github.com/openyurtio/raven/pkg/networkengine/vpndriver" "github.com/openyurtio/raven/pkg/networkengine/vpndriver/libreswan" ) +const ( + DefaultTunnelMetricsPort = 10265 + DefaultProxyMetricsPort = 10266 +) + // AgentOptions has the information that required by the raven agent type AgentOptions struct { + TunnelOptions + ProxyOptions NodeName string + NodeIP string Kubeconfig string - VPNDriver string - RouteDriver string - ForwardNodeIP bool MetricsBindAddress string } +type TunnelOptions struct { + VPNDriver string + VPNPort string + RouteDriver string + ForwardNodeIP bool +} + +type ProxyOptions struct { + ProxyMetricsAddress string + InternalInsecureAddress string + InternalSecureAddress string + ExternalAddress string + ProxyServerCertDNSNames string + ProxyServerCertIPs string + ProxyServerCertDir string + ProxyClientCertDir string + InterceptorServerUDSFile string +} + // Validate validates the AgentOptions func (o *AgentOptions) Validate() error { if o.NodeName == "" { @@ -37,6 +67,36 @@ func (o *AgentOptions) Validate() error { return errors.New("either --node-name or $NODE_NAME has to be set") } } + if o.NodeIP == "" { + o.NodeIP = os.Getenv("NODE_IP") + if o.NodeIP == "" { + return errors.New("either --node-name or $NODE_NAME has to be set") + } + } + if o.VPNPort == "" { + o.VPNPort = os.Getenv("VPN_BIND_ADDRESS") + if o.VPNPort == "" { + return errors.New("either --vpn-bind-address or $VPN_BIND_PORT has to be set") + } + } + if o.InternalSecureAddress == "" { + o.InternalSecureAddress = os.Getenv("PROXY_SERVER_INTERNAL_SECURE_ADDRESS") + if o.InternalSecureAddress == "" { + return errors.New("either --proxy-internal-secure-address or PROXY_SERVER_INTERNAL_SECURE_ADDRESS has to be set") + } + } + if o.InternalInsecureAddress == "" { + o.InternalInsecureAddress = os.Getenv("PROXY_SERVER_INTERNAL_INSECURE_ADDRESS") + if o.InternalInsecureAddress == "" { + return errors.New("either --proxy-internal-insecure-address or PROXY_SERVER_INTERNAL_INSECURE_ADDRESS has to be set") + } + } + if o.ExternalAddress == "" { + o.ExternalAddress = os.Getenv("PROXY_SERVER_EXTERNAL_ADDRESS") + if o.ExternalAddress == "" { + return errors.New("either --proxy-external-address or $PROXY_SERVER_EXTERNAL_ADDRESS has to be set") + } + } return nil } @@ -47,7 +107,17 @@ func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.VPNDriver, "vpn-driver", o.VPNDriver, `The VPN driver name. (default "libreswan")`) fs.StringVar(&o.RouteDriver, "route-driver", o.RouteDriver, `The Route driver name. (default "vxlan")`) fs.BoolVar(&o.ForwardNodeIP, "forward-node-ip", o.ForwardNodeIP, `Forward node IP or not. (default "false")`) - fs.StringVar(&o.MetricsBindAddress, "metric-bind-addr", o.MetricsBindAddress, `Binding address of metrics. (default ":8080")`) + fs.StringVar(&o.MetricsBindAddress, "metric-bind-addr", o.MetricsBindAddress, `Binding address of tunnel metrics. (default ":10265")`) + fs.StringVar(&o.VPNPort, "vpn-bind-port", o.VPNPort, `Binding port of vpn. (default ":4500")`) + fs.StringVar(&o.ProxyMetricsAddress, "proxy-metric-bind-addr", o.ProxyMetricsAddress, `Binding address of proxy metrics. (default ":10266")`) + fs.StringVar(&o.InternalSecureAddress, "proxy-internal-secure-addr", o.InternalSecureAddress, `Binding secure address of proxy server. (default ":10263")`) + fs.StringVar(&o.InternalInsecureAddress, "proxy-internal-insecure-addr", o.InternalInsecureAddress, `Binding insecure address of proxy server. (default ":10264")`) + fs.StringVar(&o.ExternalAddress, "proxy-external-addr", o.ExternalAddress, `Binding address of proxy. (default ":10262")`) + + fs.StringVar(&o.ProxyClientCertDir, "client-cert-dir", o.ProxyClientCertDir, "The directory of certificate stored at.") + fs.StringVar(&o.ProxyServerCertDir, "server-cert-dir", o.ProxyServerCertDir, "The directory of certificate stored at.") + fs.StringVar(&o.ProxyServerCertDNSNames, "server-cert-dns-names", o.ProxyServerCertDNSNames, "DNS names that will be added into server's certificate. (e.g., dns1,dns2)") + fs.StringVar(&o.ProxyServerCertIPs, "server-cert-ips", o.ProxyServerCertIPs, "IPs that will be added into server's certificate. (e.g., ip1,ip2)") } // Config return a raven agent config objective @@ -55,33 +125,71 @@ func (o *AgentOptions) Config() (*config.Config, error) { var err error c := &config.Config{ NodeName: o.NodeName, - VPNDriver: o.VPNDriver, - RouteDriver: o.RouteDriver, - ForwardNodeIP: o.ForwardNodeIP, + NodeIP: o.NodeIP, MetricsBindAddress: o.MetricsBindAddress, } cfg, err := clientcmd.BuildConfigFromFlags("", o.Kubeconfig) if err != nil { return nil, fmt.Errorf("failed to create kube client: %s", err) } - cfg = restclient.AddUserAgent(cfg, "raven-agent") - c.Kubeconfig = cfg + cfg = restclient.AddUserAgent(cfg, "raven-agent-ds") + c.KubeConfig = cfg + c.MetricsBindAddress = resolveAddress(c.MetricsBindAddress, c.NodeIP, strconv.Itoa(DefaultTunnelMetricsPort)) c.Manager, err = newMgr(cfg, c.MetricsBindAddress) if err != nil { return nil, fmt.Errorf("failed to create manager: %s", err) } - if c.VPNDriver == "" { - c.VPNDriver = libreswan.DriverName + _, port, err := net.SplitHostPort(o.VPNPort) + if err != nil { + port = strconv.Itoa(v1beta1.DefaultTunnelServerExposedPort) } - if c.RouteDriver == "" { - c.RouteDriver = vxlan.DriverName + c.Tunnel = &config.TunnelConfig{ + VPNPort: port, + VPNDriver: o.VPNDriver, + RouteDriver: o.RouteDriver, + ForwardNodeIP: o.ForwardNodeIP, } + c.Proxy = &config.ProxyConfig{ + ProxyMetricsAddress: o.ProxyMetricsAddress, + InternalInsecureAddress: o.InternalInsecureAddress, + InternalSecureAddress: o.InternalSecureAddress, + ExternalAddress: o.ExternalAddress, + } + if err != nil { + return nil, fmt.Errorf("failed to create manager: %s", err) + } + if c.Tunnel.VPNDriver == "" { + c.Tunnel.VPNDriver = libreswan.DriverName + } + if c.Tunnel.RouteDriver == "" { + c.Tunnel.RouteDriver = vxlan.DriverName + } + if c.Tunnel.VPNPort == "" { + c.Tunnel.VPNPort = vpndriver.DefaultVPNPort + } + if c.Proxy.ProxyClientCertDir == "" { + c.Proxy.ProxyClientCertDir = utils.RavenProxyClientCertDir + } + if c.Proxy.ProxyServerCertDir == "" { + c.Proxy.ProxyServerCertDir = utils.RavenProxyServerCertDir + } + if c.Proxy.InterceptorServerUDSFile == "" { + c.Proxy.InterceptorServerUDSFile = utils.RavenProxyServerUDSFile + } + + c.Proxy.InternalInsecureAddress = resolveAddress(c.Proxy.InternalInsecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerInsecurePort)) + c.Proxy.InternalSecureAddress = resolveAddress(c.Proxy.InternalSecureAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerSecurePort)) + c.Proxy.ExternalAddress = resolveAddress(c.Proxy.ExternalAddress, c.NodeIP, strconv.Itoa(v1beta1.DefaultProxyServerExposedPort)) + c.Proxy.ProxyMetricsAddress = resolveAddress(c.Proxy.ProxyMetricsAddress, c.NodeIP, strconv.Itoa(DefaultProxyMetricsPort)) + return c, err } func newMgr(cfg *restclient.Config, metricsBindAddress string) (manager.Manager, error) { scheme := runtime.NewScheme() - _ = v1alpha1.AddToScheme(scheme) + _ = v1.AddToScheme(scheme) + _ = v1beta1.AddToScheme(scheme) + opt := ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsBindAddress, @@ -94,3 +202,17 @@ func newMgr(cfg *restclient.Config, metricsBindAddress string) (manager.Manager, } return mgr, nil } + +func resolveAddress(srcAddr, defaultHost, defaultPort string) string { + host, port, err := net.SplitHostPort(srcAddr) + if err != nil { + return net.JoinHostPort(defaultHost, defaultPort) + } + if host == "" { + host = defaultHost + } + if port == "" { + port = defaultPort + } + return net.JoinHostPort(host, port) +} diff --git a/cmd/agent/app/start.go b/cmd/agent/app/start.go index 59343c5..781391b 100644 --- a/cmd/agent/app/start.go +++ b/cmd/agent/app/start.go @@ -20,15 +20,14 @@ import ( "context" "fmt" - "github.com/spf13/cobra" + "github.com/lorenzosaino/go-sysctl" "k8s.io/klog/v2" "github.com/openyurtio/raven/cmd/agent/app/config" "github.com/openyurtio/raven/cmd/agent/app/options" + ravenengine "github.com/openyurtio/raven/pkg/engine" "github.com/openyurtio/raven/pkg/features" - "github.com/openyurtio/raven/pkg/k8s" - "github.com/openyurtio/raven/pkg/networkengine/routedriver" - "github.com/openyurtio/raven/pkg/networkengine/vpndriver" + "github.com/spf13/cobra" ) // NewRavenAgentCommand creates a new raven agent command @@ -59,38 +58,29 @@ func NewRavenAgentCommand(ctx context.Context) *cobra.Command { // Run starts the raven-agent func Run(ctx context.Context, cfg *config.CompletedConfig) error { - routeDriver, err := routedriver.New(cfg.RouteDriver, cfg.Config) - if err != nil { - return fmt.Errorf("fail to create route driver: %s, %s", cfg.RouteDriver, err) - } - err = routeDriver.Init() - if err != nil { - return fmt.Errorf("fail to initialize route driver: %s, %s", cfg.RouteDriver, err) - } - klog.Infof("route driver %s initialized", cfg.RouteDriver) - vpnDriver, err := vpndriver.New(cfg.VPNDriver, cfg.Config) - if err != nil { - return fmt.Errorf("fail to create vpn driver: %s, %s", cfg.VPNDriver, err) + klog.Info("Start raven agent") + defer klog.Info("Stop raven agent") + if err := disableICMPRedirect(); err != nil { + return err } - err = vpnDriver.Init() - if err != nil { - return fmt.Errorf("fail to initialize vpn driver: %s, %s", cfg.VPNDriver, err) - } - klog.Infof("VPN driver %s initialized", cfg.VPNDriver) - // start network engine controller - ec, err := k8s.NewEngineController(cfg.NodeName, cfg.ForwardNodeIP, routeDriver, cfg.Manager, vpnDriver) - if err != nil { - return fmt.Errorf("could not create network engine controller: %s", err) - } - ec.Start(ctx) - <-ctx.Done() - err = routeDriver.Cleanup() + engine := ravenengine.NewEngine(ctx, cfg.Config) + engine.Start() + return nil +} + +func disableICMPRedirect() error { + obj := "net.ipv4.conf.all.send_redirects" + val, err := sysctl.Get(obj) if err != nil { - klog.Errorf("route driver fail to cleanup: %s", err) + klog.ErrorS(err, "failed to sysctl get", obj) + return err } - err = vpnDriver.Cleanup() - if err != nil { - klog.Errorf("vpn driver fail to cleanup: %s", err) + if val != "0" { + err = sysctl.Set(obj, "0") + if err != nil { + klog.ErrorS(err, "failed to sysctl set", obj) + return err + } } return nil } diff --git a/cmd/agent/main.go b/cmd/agent/main.go index afd54a4..fd36f50 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -22,14 +22,14 @@ import ( "time" "k8s.io/apiserver/pkg/server" + "k8s.io/klog/v2" "github.com/openyurtio/raven/cmd/agent/app" - _ "github.com/openyurtio/raven/pkg/networkengine/routedriver/vxlan" - _ "github.com/openyurtio/raven/pkg/networkengine/vpndriver/libreswan" - _ "github.com/openyurtio/raven/pkg/networkengine/vpndriver/wireguard" ) func main() { + klog.InitFlags(nil) + defer klog.Flush() rand.Seed(time.Now().UnixNano()) cmd := app.NewRavenAgentCommand(server.SetupSignalContext()) cmd.Flags().AddGoFlagSet(flag.CommandLine) diff --git a/go.mod b/go.mod index e78b0b1..dacc6f5 100644 --- a/go.mod +++ b/go.mod @@ -5,24 +5,32 @@ go 1.18 require ( github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a github.com/coreos/go-iptables v0.6.0 - github.com/openyurtio/openyurt v1.3.0 + github.com/gorilla/mux v1.8.0 + github.com/lorenzosaino/go-sysctl v0.3.1 + github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.15.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/vdobler/ht v5.3.0+incompatible github.com/vishvananda/netlink v1.2.1-beta.2 - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.12.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220504211119-3d4a969bb56b + google.golang.org/grpc v1.57.0 + k8s.io/api v0.23.2 k8s.io/apimachinery v0.23.2 k8s.io/apiserver v0.23.2 k8s.io/client-go v0.23.2 k8s.io/component-base v0.23.2 k8s.io/klog/v2 v2.30.0 + k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b + sigs.k8s.io/apiserver-network-proxy v0.28.0 sigs.k8s.io/controller-runtime v0.11.0 ) require ( + github.com/BurntSushi/toml v1.1.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -63,10 +71,10 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.15.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect go.etcd.io/etcd/api/v3 v3.5.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect @@ -85,28 +93,37 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.5.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.zx2c4.com/wireguard v0.0.0-20220407013110-ef5c587f782d // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.54.0 // indirect + google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.23.2 // indirect + honnef.co/go/tools v0.3.2 // indirect k8s.io/apiextensions-apiserver v0.23.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace ( + sigs.k8s.io/apiserver-network-proxy => github.com/openyurtio/apiserver-network-proxy v0.1.1-0.20231007082056-cecf4c454651 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 +) diff --git a/go.sum b/go.sum index 785aa45..7171522 100644 --- a/go.sum +++ b/go.sum @@ -18,14 +18,14 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= @@ -43,13 +43,22 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a h1:D9u6wYxZ2bPjDwYYq25y+n6ZmOKj/TsAMGSl4xL1yQI= github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a/go.mod h1:pzTfWeRUe2RpUHYF4s8PfLt7C3jnxg62RX10Eh9myYY= @@ -57,8 +66,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -82,6 +93,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -99,6 +111,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= @@ -122,6 +135,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -138,6 +152,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= +github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -154,6 +170,7 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -170,12 +187,16 @@ github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -188,6 +209,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -201,7 +223,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -261,15 +285,20 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -321,6 +350,7 @@ github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTx github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -345,8 +375,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lorenzosaino/go-sysctl v0.3.1 h1:3phX80tdITw2fJjZlwbXQnDWs4S30beNcMbw0cn0HtY= +github.com/lorenzosaino/go-sysctl v0.3.1/go.mod h1:5grcsBRpspKknNS1qzt1eIeRDLrhpKZAtz8Fcuvs1Rc= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= @@ -396,16 +429,20 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openyurtio/openyurt v1.3.0 h1:ItKD2HL+et4ar69CHC9qA2DOSvrHOgtDa4hnhHUQvT4= -github.com/openyurtio/openyurt v1.3.0/go.mod h1:1MDc64J8KKv57bF71l9mA5o+f1s0a8QkwEOfVmFfZhc= +github.com/openyurtio/apiserver-network-proxy v0.1.1-0.20231007082056-cecf4c454651 h1:pBjrj95WZ7NkaIQvvb7hveI5fd8fXxlX7OtKRcciamY= +github.com/openyurtio/apiserver-network-proxy v0.1.1-0.20231007082056-cecf4c454651/go.mod h1:yPrw5zKs7BVVCuQUaY4MzSmsYPKXWnZsrjKUWQrZX3w= +github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167 h1:gbjh0njlthM0mM9Z6LHURdKpovLXReleT5GUwaksyFE= +github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167/go.mod h1:c77rs9s+ztXyyHNf2EB+lMHuDfSWwel6dhP2ZVi5cvM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -424,8 +461,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -451,7 +488,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -475,12 +513,14 @@ github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -495,8 +535,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= @@ -579,16 +619,18 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -599,6 +641,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d h1:+W8Qf4iJtMGKkyAygcKohjxTk4JPsL9DpzApJ22m5Ic= +golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -612,6 +656,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -624,6 +669,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -643,6 +691,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -673,8 +722,8 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -689,8 +738,8 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -702,8 +751,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -711,6 +761,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -723,6 +774,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -772,13 +824,14 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -788,8 +841,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -799,6 +852,7 @@ golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -855,6 +909,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -940,8 +996,12 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -964,8 +1024,8 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1023,48 +1083,62 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= +honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= +k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.2 h1:62cpzreV3dCuj0hqPi8r4dyWh48ogMcyh+ga9jEGij4= k8s.io/api v0.23.2/go.mod h1:sYuDb3flCtRPI8ghn6qFrcK5ZBu2mhbElxRE95qpwlI= k8s.io/apiextensions-apiserver v0.23.0 h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY= k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4= +k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.2 h1:dBmjCOeYBdg2ibcQxMuUq+OopZ9fjfLIR5taP/XKeTs= k8s.io/apimachinery v0.23.2/go.mod h1:zDqeV0AK62LbCI0CI7KbWCAYdLg+E+8UXJ0rIz5gmS8= k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4= k8s.io/apiserver v0.23.2 h1:vGFCojjwSLyunapA7FWuzyekml/s0nAsoh4iBpzWzOs= k8s.io/apiserver v0.23.2/go.mod h1:Kdt8gafkPev9Gfh+H6lCPbmRu42f7BfhOfHKKa3dtyU= +k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= k8s.io/client-go v0.23.2 h1:BNbOcxa99jxHH8mM1cPKGIrrKRnCSAfAtyonYGsbFtE= k8s.io/client-go v0.23.2/go.mod h1:k3YbsWg6GWdHF1THHTQP88X9RhB1DWPo3Dq7KfU/D1c= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= +k8s.io/component-base v0.18.8/go.mod h1:00frPRDas29rx58pPCxNkhUfPbwajlyyvu8ruNgSErU= k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI= k8s.io/component-base v0.23.2 h1:dAYmUhWIBWO762etTjBEEKtYYHi5CoQInSLtK6LM1Zs= k8s.io/component-base v0.23.2/go.mod h1:wS9Z03MO3oJ0RU8bB/dbXTiluGju+SC/F5i660gxB8c= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27 h1:KQOkVzXrLNb0EP6W0FD6u3CCPAwgXFYwZitbj7K0P0Y= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27/go.mod h1:tq2nT0Kx7W+/f2JVE+zxYtUhdjuELJkVpNz+x/QN5R4= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ= sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go new file mode 100644 index 0000000..34208e5 --- /dev/null +++ b/pkg/engine/engine.go @@ -0,0 +1,139 @@ +package engine + +import ( + "context" + "time" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/raven/cmd/agent/app/config" + "github.com/openyurtio/raven/pkg/utils" +) + +type Engine struct { + nodeName string + nodeIP string + context context.Context + manager manager.Manager + client client.Client + option StatusOption + + tunnelQueue workqueue.RateLimitingInterface + tunnelEngine *TunnelEngine + + proxyQueue workqueue.RateLimitingInterface + proxyEngine *ProxyEngine +} + +func NewEngine(ctx context.Context, cfg *config.Config) *Engine { + engine := &Engine{ + nodeName: cfg.NodeName, + nodeIP: cfg.NodeIP, + manager: cfg.Manager, + context: ctx, + option: NewEngineOption(), + tunnelQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tunnel"), + proxyQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Proxy"), + } + err := ctrl.NewControllerManagedBy(engine.manager). + For(&v1beta1.Gateway{}, builder.WithPredicates(predicate.Funcs{ + CreateFunc: engine.addGateway, + UpdateFunc: engine.updateGateway, + DeleteFunc: engine.deleteGateway, + })). + Complete(reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + })) + if err != nil { + klog.ErrorS(err, utils.FormatRavenEngine("failed to new raven agent controller with manager")) + } + engine.client = engine.manager.GetClient() + engine.tunnelEngine = newTunnelEngine(cfg, engine.client, engine.option, engine.tunnelQueue) + engine.proxyEngine = newProxyEngine(engine.context, cfg, engine.client, engine.option, engine.proxyQueue) + return engine +} + +func (e *Engine) Start() { + defer utilruntime.HandleCrash() + klog.Info(utils.FormatRavenEngine("engine successfully start")) + go func() { + if err := e.manager.Start(e.context); err != nil { + klog.ErrorS(err, utils.FormatRavenEngine("failed to start engine controller")) + } + }() + go wait.Until(e.tunnelEngine.worker, time.Second, e.context.Done()) + go wait.Until(e.proxyEngine.worker, time.Second, e.context.Done()) + <-e.context.Done() + e.cleanup() + klog.Info(utils.FormatRavenEngine("engine successfully stop")) +} + +func (e *Engine) cleanup() { + if e.option.GetTunnelStatus() { + err := e.tunnelEngine.clearDriver() + if err != nil { + klog.Errorf(utils.FormatRavenEngine("failed to cleanup tunnel driver, error %s", err.Error())) + } + } + if e.option.GetProxyStatus() { + e.proxyEngine.stopServers() + } +} + +func (e *Engine) enqueueTunnel(obj *v1beta1.Gateway) { + klog.Info(utils.FormatRavenEngine("enqueue gateway %s to tunnel queue", obj.Name)) + e.tunnelQueue.Add(obj) +} + +func (e *Engine) enqueueProxy(obj *v1beta1.Gateway) { + klog.Info(utils.FormatRavenEngine("enqueue gateway %s to proxy queue", obj.Name)) + e.proxyQueue.Add(obj) +} + +func (e *Engine) addGateway(evt event.CreateEvent) bool { + gw, ok := evt.Object.(*v1beta1.Gateway) + if ok { + klog.InfoS(utils.FormatRavenEngine("adding gateway %s", gw.GetName())) + e.enqueueTunnel(gw.DeepCopy()) + e.enqueueProxy(gw.DeepCopy()) + } + return ok +} + +func (e *Engine) updateGateway(evt event.UpdateEvent) bool { + oldGw, ok1 := evt.ObjectOld.(*v1beta1.Gateway) + newGw, ok2 := evt.ObjectNew.(*v1beta1.Gateway) + update := false + if ok1 && ok2 { + if oldGw.ResourceVersion != newGw.ResourceVersion { + update = true + klog.InfoS(utils.FormatRavenEngine("updating gateway, %s", newGw.GetName())) + e.enqueueTunnel(newGw.DeepCopy()) + e.enqueueProxy(newGw.DeepCopy()) + } else { + klog.InfoS(utils.FormatRavenEngine("skip handle update gateway"), klog.KObj(newGw)) + } + } + return update +} + +func (e *Engine) deleteGateway(evt event.DeleteEvent) bool { + gw, ok := evt.Object.(*v1beta1.Gateway) + if ok { + klog.InfoS(utils.FormatRavenEngine("deleting gateway, %s", gw.GetName())) + e.enqueueTunnel(gw.DeepCopy()) + e.enqueueProxy(gw.DeepCopy()) + } + return ok +} diff --git a/pkg/engine/proxy.go b/pkg/engine/proxy.go new file mode 100644 index 0000000..12d5170 --- /dev/null +++ b/pkg/engine/proxy.go @@ -0,0 +1,300 @@ +package engine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net" + "sort" + "strconv" + "strings" + "time" + + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + ravenutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" + "github.com/openyurtio/raven/cmd/agent/app/config" + "github.com/openyurtio/raven/pkg/proxyengine" + "github.com/openyurtio/raven/pkg/proxyengine/proxyclient" + "github.com/openyurtio/raven/pkg/proxyengine/proxyserver" + "github.com/openyurtio/raven/pkg/utils" +) + +type ProxyEngine struct { + nodeName string + nodeIP string + proxyServerAddresses []string + gateway *v1beta1.Gateway + config *config.Config + client client.Client + engineOption StatusOption + proxyOption *proxyOption + ravenContext context.Context + proxyContext ProxyContext + queue workqueue.RateLimitingInterface +} + +func newProxyEngine(ctx context.Context, cfg *config.Config, client client.Client, opt StatusOption, queue workqueue.RateLimitingInterface) *ProxyEngine { + return &ProxyEngine{ + nodeName: cfg.NodeName, + nodeIP: cfg.NodeIP, + config: cfg, + client: client, + engineOption: opt, + ravenContext: ctx, + proxyOption: newProxyOption(), + proxyContext: newProxyContext(ctx), + queue: queue, + } +} + +func (p *ProxyEngine) worker() { + for p.processNextWorkItem() { + } +} + +func (p *ProxyEngine) processNextWorkItem() bool { + obj, quit := p.queue.Get() + if quit { + return false + } + gw, ok := obj.(*v1beta1.Gateway) + if !ok { + return false + } + defer p.queue.Done(gw) + + err := p.handler(gw) + p.handleEventErr(err, gw) + return true +} + +func (p *ProxyEngine) handler(gw *v1beta1.Gateway) error { + curServer := p.proxyOption.GetServerStatus() + curClient := p.proxyOption.GetClientStatus() + specServer, specClient := p.getRole(enableProxy(gw)) + var err error + p.gateway, err = utils.GetOwnGateway(p.client, p.nodeName) + if err != nil { + klog.Errorf("failed get gateway for %s, can not start proxy server", p.nodeName) + return fmt.Errorf("failed get gateway name for %s, can not start proxy server", p.nodeName) + } + if !curServer && specServer { + klog.Infoln(utils.FormatProxyServer("start raven l7 proxy server")) + if p.gateway == nil { + klog.Errorf("unknown gateway for node %s, can not start proxy server", p.nodeName) + } + pe := &proxyengine.EnginConfig{ + Name: p.nodeName, + IP: p.nodeIP, + GatewayName: p.gateway.Name, + CertDir: p.config.Proxy.ProxyServerCertDir, + MetaAddress: p.config.Proxy.ProxyMetricsAddress, + CertIPs: p.config.Proxy.ProxyServerCertIPs, + CertDNSNames: p.config.Proxy.ProxyServerCertDNSNames, + InterceptorUDSFile: p.config.Proxy.InterceptorServerUDSFile, + InternalSecureAddress: p.config.Proxy.InternalSecureAddress, + InternalInsecureAddress: p.config.Proxy.InternalInsecureAddress, + ExposedAddress: p.config.Proxy.ExternalAddress, + } + ctx := p.proxyContext.GetServerContext() + ps, err := proxyserver.NewProxyServer(pe, p.client, p.config.Manager.GetConfig(), p.gateway.DeepCopy()) + if err != nil { + klog.Errorf("failed to new proxy server, error %s", err.Error()) + return err + } + err = ps.Start(ctx) + if err != nil { + klog.Errorf("failed to start proxy server, error %s", err.Error()) + } + p.proxyOption.SetServerStatus(specServer) + } else if curServer && !specServer { + klog.Infoln(utils.FormatProxyServer("Stop raven l7 proxy server")) + cancel := p.proxyContext.GetServerCancelFunc() + cancel() + p.proxyOption.SetServerStatus(specServer) + p.proxyContext.ReloadServerContext(p.ravenContext) + } + + if !curClient && specClient { + klog.Infoln(utils.FormatProxyClient("start raven l7 proxy client")) + var err error + dstAddr := getDestAddressForProxyClient(p.client, p.gateway) + if len(dstAddr) < 1 { + klog.Infoln(utils.FormatProxyClient("dest address is empty, will not connected it")) + return nil + } + p.proxyServerAddresses = dstAddr + pe := &proxyengine.EnginConfig{ + Name: p.nodeName, + IP: p.nodeIP, + CertDir: p.config.Proxy.ProxyClientCertDir, + MetaAddress: p.config.Proxy.ProxyMetricsAddress, + } + pc, err := proxyclient.NewProxyClient(pe, p.proxyServerAddresses, p.config.KubeConfig) + if err != nil { + klog.Errorf("failed to new proxy client, error %s", err.Error()) + return err + } + ctx := p.proxyContext.GetClientContext() + err = pc.Start(ctx) + if err != nil { + klog.Errorf("failed to start proxy client, error %s", err.Error()) + } + p.proxyOption.SetClientStatus(specClient) + return nil + } else if curClient && !specClient { + klog.Infoln(utils.FormatProxyClient("stop raven l7 proxy client")) + cancel := p.proxyContext.GetClientCancelFunc() + cancel() + p.proxyOption.SetClientStatus(specClient) + p.proxyContext.ReloadClientContext(p.ravenContext) + } else if curClient && specClient { + dstAddr := getDestAddressForProxyClient(p.client, p.gateway) + if len(dstAddr) < 1 { + klog.Infoln(utils.FormatProxyClient("dest address is empty, will not connected it")) + return nil + } + if computeHash(strings.Join(p.proxyServerAddresses, ",")) != computeHash(strings.Join(dstAddr, ",")) { + klog.Infoln(utils.FormatProxyClient("Update raven l7 proxy client")) + cancel := p.proxyContext.GetClientCancelFunc() + cancel() + time.Sleep(2 * time.Second) + p.proxyContext.ReloadClientContext(p.ravenContext) + p.proxyServerAddresses = dstAddr + pe := &proxyengine.EnginConfig{ + Name: p.nodeName, + IP: p.nodeIP, + CertDir: p.config.Proxy.ProxyClientCertDir, + MetaAddress: p.config.Proxy.ProxyMetricsAddress, + } + pc, err := proxyclient.NewProxyClient(pe, p.proxyServerAddresses, p.config.KubeConfig) + if err != nil { + klog.Errorf("failed to new proxy server, error %s", err.Error()) + return err + } + ctx := p.proxyContext.GetClientContext() + err = pc.Start(ctx) + if err != nil { + klog.Errorf("failed to start proxy client, error %s", err.Error()) + } + } + } + return nil +} + +func getDestAddressForProxyClient(client client.Client, ownGateway *v1beta1.Gateway) []string { + destAddr := make([]string, 0) + var gwList v1beta1.GatewayList + err := client.List(context.TODO(), &gwList) + if err != nil { + return destAddr + } + for _, gw := range gwList.Items { + if gw.Spec.ExposeType == "" { + continue + } + if ownGateway != nil && ownGateway.Name == gw.Name { + continue + } + for _, aep := range gw.Status.ActiveEndpoints { + if aep.Type == v1beta1.Proxy && aep.PublicIP != "" { + destAddr = append(destAddr, net.JoinHostPort(aep.PublicIP, strconv.Itoa(aep.Port))) + } + } + } + sort.Slice(destAddr, func(i, j int) bool { return destAddr[i] < destAddr[j] }) + return destAddr +} + +func (p *ProxyEngine) getRole(enableProxy bool) (enableServer, enableClient bool) { + enableServer = false + enableClient = false + if !enableProxy { + return + } + var gwList v1beta1.GatewayList + err := p.client.List(p.ravenContext, &gwList) + if err != nil { + return + } + + for _, gw := range gwList.Items { + for _, aep := range gw.Status.ActiveEndpoints { + if aep.NodeName == p.nodeName && aep.Type == v1beta1.Proxy { + enableClient = true + if gw.Spec.ExposeType != "" { + enableServer = true + } else { + enableServer = false + } + return + } + } + for _, node := range gw.Status.Nodes { + if node.NodeName == p.nodeName { + enableServer = false + enableClient = false + return + } + } + } + enableServer = false + enableClient = true + return +} + +func (p *ProxyEngine) stopServers() { + if p.proxyOption.GetServerStatus() { + cancelServer := p.proxyContext.GetServerCancelFunc() + cancelServer() + } + if p.proxyOption.GetClientStatus() { + cancelClient := p.proxyContext.GetClientCancelFunc() + cancelClient() + } +} + +func enableProxy(gw *v1beta1.Gateway) (enable bool) { + enable = false + for _, aep := range gw.Status.ActiveEndpoints { + if aep.Type == v1beta1.Proxy { + if aep.Config == nil { + enable = false + return + } + start, ok := aep.Config[ravenutil.RavenEnableProxy] + if !ok { + enable = false + return + } + if strings.ToLower(start) == "true" { + enable = true + } + } + } + return +} + +func (p *ProxyEngine) handleEventErr(err error, event interface{}) { + if err == nil { + p.queue.Forget(event) + return + } + if p.queue.NumRequeues(event) < utils.MaxRetries { + klog.Infof("error syncing event %v: %v", event, err) + p.queue.AddRateLimited(event) + return + } + klog.Infof("dropping event %q out of the queue: %v", event, err) + p.queue.Forget(event) +} + +func computeHash(target string) string { + hash := sha256.Sum224([]byte(target)) + return strings.ToLower(hex.EncodeToString(hash[:])) +} diff --git a/pkg/engine/tunnel.go b/pkg/engine/tunnel.go new file mode 100644 index 0000000..9b39558 --- /dev/null +++ b/pkg/engine/tunnel.go @@ -0,0 +1,126 @@ +package engine + +import ( + "fmt" + + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/raven/cmd/agent/app/config" + "github.com/openyurtio/raven/pkg/networkengine/routedriver" + "github.com/openyurtio/raven/pkg/networkengine/vpndriver" + "github.com/openyurtio/raven/pkg/tunnelengine" + "github.com/openyurtio/raven/pkg/utils" +) + +type TunnelEngine struct { + nodeName string + config *config.Config + client client.Client + option StatusOption + queue workqueue.RateLimitingInterface + routeDriver routedriver.Driver + vpnDriver vpndriver.Driver + tunnelHandler *tunnelengine.TunnelHandler +} + +func newTunnelEngine(cfg *config.Config, client client.Client, opt StatusOption, queue workqueue.RateLimitingInterface) *TunnelEngine { + return &TunnelEngine{nodeName: cfg.NodeName, config: cfg, client: client, option: opt, queue: queue} +} + +func (t *TunnelEngine) worker() { + for t.processNextWorkItem() { + } +} + +func (t *TunnelEngine) processNextWorkItem() bool { + obj, quit := t.queue.Get() + if quit { + return false + } + gw, ok := obj.(*v1beta1.Gateway) + if !ok { + return false + } + defer t.queue.Done(gw) + err := t.handler(gw) + t.handleEventErr(err, gw) + return true +} + +func (t *TunnelEngine) handler(gw *v1beta1.Gateway) error { + klog.Info(utils.FormatRavenEngine("update raven l3 tunnel config for gateway %s", gw.GetName())) + err := t.reconcile() + if err != nil { + klog.Errorf("failed update tunnel driver, error %s", err.Error()) + return err + } + return nil +} + +func (t *TunnelEngine) initDriver() error { + routeDriver, err := routedriver.New(t.config.Tunnel.RouteDriver, t.config) + if err != nil { + return fmt.Errorf("fail to create route driver: %s, %s", t.config.Tunnel.RouteDriver, err) + } + err = routeDriver.Init() + if err != nil { + return fmt.Errorf("fail to initialize route driver: %s, %s", t.config.Tunnel.RouteDriver, err) + } + t.routeDriver = routeDriver + klog.Info(utils.FormatRavenEngine("route driver %s initialized", t.config.Tunnel.RouteDriver)) + vpnDriver, err := vpndriver.New(t.config.Tunnel.VPNDriver, t.config) + if err != nil { + return fmt.Errorf("fail to create vpn driver: %s, %s", t.config.Tunnel.VPNDriver, err) + } + err = vpnDriver.Init() + if err != nil { + return fmt.Errorf("fail to initialize vpn driver: %s, %s", t.config.Tunnel.VPNDriver, err) + } + klog.Info(utils.FormatRavenEngine("VPN driver %s initialized", t.config.Tunnel.VPNDriver)) + t.vpnDriver = vpnDriver + t.tunnelHandler = tunnelengine.NewTunnelHandler(t.nodeName, t.config.Tunnel.ForwardNodeIP, t.client, t.routeDriver, t.vpnDriver) + return nil +} + +func (t *TunnelEngine) clearDriver() error { + err := t.routeDriver.Cleanup() + if err != nil { + klog.Errorf(utils.FormatRavenEngine("fail to cleanup route driver: %s", err.Error())) + } + err = t.vpnDriver.Cleanup() + if err != nil { + klog.Errorf(utils.FormatRavenEngine("fail to cleanup vpn driver: %s", err.Error())) + } + return nil +} + +func (t *TunnelEngine) reconcile() error { + if t.routeDriver == nil || t.vpnDriver == nil { + err := t.initDriver() + if err != nil { + klog.Errorf(utils.FormatRavenEngine("failed to init raven l3 tunnel engine")) + } + } + err := t.tunnelHandler.Handler() + if err != nil { + return err + } + return nil +} + +func (t *TunnelEngine) handleEventErr(err error, event interface{}) { + if err == nil { + t.queue.Forget(event) + return + } + if t.queue.NumRequeues(event) < utils.MaxRetries { + klog.Info(utils.FormatRavenEngine("error syncing event %v: %v", event, err)) + t.queue.AddRateLimited(event) + return + } + klog.Info(utils.FormatRavenEngine("dropping event %q out of the queue: %v", event, err)) + t.queue.Forget(event) +} diff --git a/pkg/engine/utils.go b/pkg/engine/utils.go new file mode 100644 index 0000000..de9e829 --- /dev/null +++ b/pkg/engine/utils.go @@ -0,0 +1,140 @@ +package engine + +import ( + "context" + "sync" +) + +type StatusOption interface { + GetProxyStatus() bool + GetTunnelStatus() bool + SetProxyStatus(status bool) + SetTunnelStatus(status bool) +} + +type engineOption struct { + mu sync.Mutex + enableProxy bool + enableTunnel bool +} + +func NewEngineOption() StatusOption { + return &engineOption{enableTunnel: false, enableProxy: false} +} + +func (s *engineOption) GetProxyStatus() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.enableProxy +} + +func (s *engineOption) GetTunnelStatus() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.enableTunnel +} + +func (s *engineOption) SetProxyStatus(status bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.enableProxy = status +} + +func (s *engineOption) SetTunnelStatus(status bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.enableTunnel = status +} + +type proxyOption struct { + mu sync.Mutex + enableClient bool + enableServer bool +} + +func newProxyOption() *proxyOption { + return &proxyOption{enableClient: false, enableServer: false} +} + +func (s *proxyOption) GetClientStatus() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.enableClient +} + +func (s *proxyOption) GetServerStatus() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.enableServer +} + +func (s *proxyOption) SetClientStatus(enable bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.enableClient = enable +} + +func (s *proxyOption) SetServerStatus(enable bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.enableServer = enable +} + +type ProxyContext interface { + ReloadClientContext(ctx context.Context) + ReloadServerContext(ctx context.Context) + GetClientContext() context.Context + GetServerContext() context.Context + GetClientCancelFunc() context.CancelFunc + GetServerCancelFunc() context.CancelFunc +} + +type proxyContexts struct { + mu sync.Mutex + clientContext context.Context + serverContext context.Context + clientCancel context.CancelFunc + serverCancel context.CancelFunc +} + +func newProxyContext(ctx context.Context) ProxyContext { + clientCtx, clientCanc := context.WithCancel(ctx) + serverCtx, serverCanc := context.WithCancel(ctx) + return &proxyContexts{clientContext: clientCtx, clientCancel: clientCanc, serverContext: serverCtx, serverCancel: serverCanc} +} + +func (r *proxyContexts) ReloadClientContext(ctx context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + r.clientContext, r.clientCancel = context.WithCancel(ctx) +} + +func (r *proxyContexts) ReloadServerContext(ctx context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + r.serverContext, r.serverCancel = context.WithCancel(ctx) +} + +func (r *proxyContexts) GetClientContext() context.Context { + r.mu.Lock() + defer r.mu.Unlock() + return r.clientContext +} + +func (r *proxyContexts) GetServerContext() context.Context { + r.mu.Lock() + defer r.mu.Unlock() + return r.serverContext +} + +func (r *proxyContexts) GetClientCancelFunc() context.CancelFunc { + r.mu.Lock() + defer r.mu.Unlock() + return r.clientCancel +} + +func (r *proxyContexts) GetServerCancelFunc() context.CancelFunc { + r.mu.Lock() + defer r.mu.Unlock() + return r.serverCancel +} diff --git a/pkg/k8s/engine_controller.go b/pkg/k8s/engine_controller.go deleted file mode 100644 index e78219d..0000000 --- a/pkg/k8s/engine_controller.go +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright 2022 The OpenYurt Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package k8s - -import ( - "context" - "net" - "reflect" - "time" - - "github.com/EvilSuperstars/go-cidrman" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/openyurtio/raven/pkg/networkengine/routedriver" - "github.com/openyurtio/raven/pkg/networkengine/vpndriver" - "github.com/openyurtio/raven/pkg/types" - "github.com/openyurtio/raven/pkg/utils" -) - -const ( - maxRetries = 30 -) - -type EngineController struct { - nodeName string - forwardNodeIP bool - nodeInfos map[types.NodeName]*v1alpha1.NodeInfo - network *types.Network - // lastSeenNetwork tracks the last seen Network. - lastSeenNetwork *types.Network - - manager manager.Manager - - ravenClient client.Client - queue workqueue.RateLimitingInterface - - routeDriver routedriver.Driver - vpnDriver vpndriver.Driver -} - -func NewEngineController(nodeName string, forwardNodeIP bool, routeDriver routedriver.Driver, manager manager.Manager, - vpnDriver vpndriver.Driver) (*EngineController, error) { - ctr := &EngineController{ - nodeName: nodeName, - forwardNodeIP: forwardNodeIP, - queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), - routeDriver: routeDriver, - manager: manager, - vpnDriver: vpnDriver, - } - - err := ctrl.NewControllerManagedBy(ctr.manager). - For(&v1alpha1.Gateway{}, builder.WithPredicates(predicate.Funcs{ - CreateFunc: ctr.addGateway, - UpdateFunc: ctr.updateGateway, - DeleteFunc: ctr.deleteGateway, - })). - Complete(reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil - })) - if err != nil { - klog.ErrorS(err, "failed to new raven agent controller with manager") - } - ctr.ravenClient = ctr.manager.GetClient() - - return ctr, nil -} - -func (c *EngineController) Start(ctx context.Context) { - defer utilruntime.HandleCrash() - go func() { - if err := c.manager.Start(ctx); err != nil { - klog.ErrorS(err, "failed to start engine controller") - } - }() - go wait.Until(c.worker, time.Second, ctx.Done()) - klog.Info("engine controller successfully start") -} - -func (c *EngineController) worker() { - for c.processNextWorkItem() { - } -} - -func (c *EngineController) enqueue(obj *v1alpha1.Gateway) { - c.queue.Add(obj.Name) -} - -func (c *EngineController) processNextWorkItem() bool { - key, quit := c.queue.Get() - if quit { - return false - } - defer c.queue.Done(key) - - err := c.sync() - c.handleEventErr(err, key) - - return true -} - -func (c *EngineController) getMergedSubnets(nodeInfo []v1alpha1.NodeInfo) []string { - subnets := make([]string, 0) - for _, n := range nodeInfo { - subnets = append(subnets, n.Subnets...) - } - subnets, _ = cidrman.MergeCIDRs(subnets) - return subnets -} - -// sync syncs full state according to the gateway list. -func (c *EngineController) sync() error { - var gws v1alpha1.GatewayList - err := c.ravenClient.List(context.Background(), &gws) - if err != nil { - return err - } - // As we are going to rebuild a full state, so cleanup before proceeding. - c.network = &types.Network{ - LocalEndpoint: nil, - RemoteEndpoints: make(map[types.GatewayName]*types.Endpoint), - LocalNodeInfo: make(map[types.NodeName]*v1alpha1.NodeInfo), - RemoteNodeInfo: make(map[types.NodeName]*v1alpha1.NodeInfo), - } - c.nodeInfos = make(map[types.NodeName]*v1alpha1.NodeInfo) - - for i := range gws.Items { - // try to update public IP if empty. - gw := &gws.Items[i] - if ep := gw.Status.ActiveEndpoint; ep != nil && ep.PublicIP == "" { - err := c.configGatewayPublicIP(gw) - if err != nil { - klog.ErrorS(err, "error config gateway public ip", "gateway", klog.KObj(gw)) - } - continue - } - if !c.shouldHandleGateway(gw) { - continue - } - c.syncNodeInfo(gw.Status.Nodes) - } - for i := range gws.Items { - gw := &gws.Items[i] - if !c.shouldHandleGateway(gw) { - continue - } - c.syncGateway(gw) - } - if reflect.DeepEqual(c.network, c.lastSeenNetwork) { - klog.Info("network not changed, skip to process") - return nil - } - nw := c.network.Copy() - klog.InfoS("applying network", "localEndpoint", nw.LocalEndpoint, "remoteEndpoint", nw.RemoteEndpoints) - err = c.vpnDriver.Apply(nw, c.routeDriver.MTU) - if err != nil { - return err - } - err = c.routeDriver.Apply(nw, c.vpnDriver.MTU) - if err != nil { - return err - } - - // Only update lastSeenNetwork when all operations succeeded. - c.lastSeenNetwork = c.network - return nil -} - -func (c *EngineController) syncNodeInfo(nodes []v1alpha1.NodeInfo) { - for _, v := range nodes { - c.nodeInfos[types.NodeName(v.NodeName)] = v.DeepCopy() - } -} - -func (c *EngineController) appendNodeIP(gw *v1alpha1.Gateway) { - for i := range gw.Status.Nodes { - nodeSubnet := net.IPNet{ - IP: net.ParseIP(gw.Status.Nodes[i].PrivateIP), - Mask: []byte{0xff, 0xff, 0xff, 0xff}, - } - gw.Status.Nodes[i].Subnets = append(gw.Status.Nodes[i].Subnets, nodeSubnet.String()) - } -} - -func (c *EngineController) syncGateway(gw *v1alpha1.Gateway) { - if c.forwardNodeIP { - c.appendNodeIP(gw) - } - aep := gw.Status.ActiveEndpoint - subnets := c.getMergedSubnets(gw.Status.Nodes) - cfg := make(map[string]string) - for k := range aep.Config { - cfg[k] = aep.Config[k] - } - var nodeInfo *v1alpha1.NodeInfo - if nodeInfo = c.nodeInfos[types.NodeName(aep.NodeName)]; nodeInfo == nil { - klog.Errorf("node %s is found in Endpoint but not existed in NodeInfo", aep.NodeName) - return - } - ep := &types.Endpoint{ - GatewayName: types.GatewayName(gw.Name), - NodeName: types.NodeName(aep.NodeName), - Subnets: subnets, - PrivateIP: nodeInfo.PrivateIP, - PublicIP: aep.PublicIP, - UnderNAT: aep.UnderNAT, - Config: cfg, - } - var isLocalGateway bool - defer func() { - for _, v := range gw.Status.Nodes { - if isLocalGateway { - c.network.LocalNodeInfo[types.NodeName(v.NodeName)] = v.DeepCopy() - } else { - c.network.RemoteNodeInfo[types.NodeName(v.NodeName)] = v.DeepCopy() - } - } - }() - for _, v := range gw.Status.Nodes { - if v.NodeName == c.nodeName { - c.network.LocalEndpoint = ep - isLocalGateway = true - return - } - } - c.network.RemoteEndpoints[types.GatewayName(gw.Name)] = ep -} - -func (c *EngineController) handleEventErr(err error, event interface{}) { - if err == nil { - c.queue.Forget(event) - return - } - if c.queue.NumRequeues(event) < maxRetries { - klog.Infof("error syncing event %v: %v", event, err) - c.queue.AddRateLimited(event) - return - } - - utilruntime.HandleError(err) - klog.Infof("dropping event %q out of the queue: %v", event, err) - c.queue.Forget(event) -} - -func (c *EngineController) shouldHandleGateway(gateway *v1alpha1.Gateway) bool { - if gateway.Status.ActiveEndpoint == nil { - klog.InfoS("no active endpoint , waiting for sync", "gateway", klog.KObj(gateway)) - return false - } - if gateway.Status.ActiveEndpoint.PublicIP == "" { - klog.InfoS("no public IP for gateway, waiting for sync", "gateway", klog.KObj(gateway)) - return false - } - return true -} - -func (c *EngineController) configGatewayPublicIP(gateway *v1alpha1.Gateway) error { - if gateway.Status.ActiveEndpoint.NodeName != c.nodeName { - return nil - } - - publicIP, err := utils.GetPublicIP() - if err != nil { - return err - } - - // retry to update public ip of localGateway - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - // get localGateway from api server - var apiGw v1alpha1.Gateway - err := c.ravenClient.Get(context.Background(), client.ObjectKey{ - Name: gateway.Name, - }, &apiGw) - if err != nil { - return err - } - for k, v := range apiGw.Spec.Endpoints { - if v.NodeName == c.nodeName { - apiGw.Spec.Endpoints[k].PublicIP = publicIP - err = c.ravenClient.Update(context.Background(), &apiGw) - return err - } - } - return nil - }) - return err -} - -func (c *EngineController) addGateway(e event.CreateEvent) bool { - gw, ok := e.Object.(*v1alpha1.Gateway) - if ok { - klog.V(4).InfoS("adding gateway", "gateway", klog.KObj(gw)) - c.enqueue(gw) - } - return ok -} - -func (c *EngineController) updateGateway(e event.UpdateEvent) bool { - oldGw, ok1 := e.ObjectOld.(*v1alpha1.Gateway) - newGw, ok2 := e.ObjectNew.(*v1alpha1.Gateway) - update := false - if ok1 && ok2 { - if oldGw.ResourceVersion != newGw.ResourceVersion { - update = true - klog.V(4).InfoS("updating gateway", "gateway", klog.KObj(newGw)) - c.enqueue(newGw) - } - klog.InfoS("skip handle update gateway", "gateway", klog.KObj(newGw)) - } - return update -} - -func (c *EngineController) deleteGateway(e event.DeleteEvent) bool { - gw, ok := e.Object.(*v1alpha1.Gateway) - if ok { - klog.InfoS("deleting gateway", "gateway", klog.KObj(gw)) - c.enqueue(gw) - } - return ok -} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..d3534b4 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,95 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +var ( + namespace = strings.ReplaceAll(projectinfo.GetTunnelName(), "-", "_") + subsystem = "server" +) + +var ( + // Metrics provides access to all tunnel server metrics. + Metrics = newRavenProxyMetrics() +) + +type RavenProxyMetrics struct { + proxyingRequestsCollector *prometheus.GaugeVec + proxyingRequestsGauge prometheus.Gauge + cloudNodeGauge prometheus.Gauge +} + +func newRavenProxyMetrics() *RavenProxyMetrics { + proxyingRequestsCollector := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "in_proxy_requests", + Help: "how many http requests are proxying by raven proxy server", + }, + []string{"verb", "path"}) + proxyingRequestsGauge := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "total_in_proxy_requests", + Help: "the number of http requests are proxying by raven proxy server", + }) + cloudNodeGauge := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "cloud_nodes_counter", + Help: "counter of cloud nodes that do not run raven proxy client", + }) + + prometheus.MustRegister(proxyingRequestsCollector) + prometheus.MustRegister(proxyingRequestsGauge) + prometheus.MustRegister(cloudNodeGauge) + return &RavenProxyMetrics{ + proxyingRequestsCollector: proxyingRequestsCollector, + proxyingRequestsGauge: proxyingRequestsGauge, + cloudNodeGauge: cloudNodeGauge, + } +} + +func (tsm *RavenProxyMetrics) Reset() { + tsm.proxyingRequestsCollector.Reset() + tsm.proxyingRequestsGauge.Set(float64(0)) + tsm.cloudNodeGauge.Set(float64(0)) +} + +func (tsm *RavenProxyMetrics) IncInFlightRequests(verb, path string) { + tsm.proxyingRequestsCollector.WithLabelValues(verb, path).Inc() + tsm.proxyingRequestsGauge.Inc() +} + +func (tsm *RavenProxyMetrics) DecInFlightRequests(verb, path string) { + tsm.proxyingRequestsCollector.WithLabelValues(verb, path).Dec() + tsm.proxyingRequestsGauge.Dec() +} + +func (tsm *RavenProxyMetrics) ObserveCloudNodes(cnt int) { + tsm.cloudNodeGauge.Set(float64(cnt)) +} diff --git a/pkg/networkengine/routedriver/driver.go b/pkg/networkengine/routedriver/driver.go index 9cbc325..65cf526 100644 --- a/pkg/networkengine/routedriver/driver.go +++ b/pkg/networkengine/routedriver/driver.go @@ -9,6 +9,7 @@ import ( "github.com/openyurtio/raven/cmd/agent/app/config" "github.com/openyurtio/raven/pkg/types" + "github.com/openyurtio/raven/pkg/utils" ) // Driver is the interface for inner gateway routing mechanism. @@ -44,9 +45,9 @@ func RegisterRouteDriver(name string, factory Factory) { driversMutex.Lock() defer driversMutex.Unlock() if _, found := drivers[name]; found { - klog.Fatalf("route drivers %q was registered twice", name) + klog.Fatal(utils.FormatTunnel("route drivers %q was registered twice", name)) } - klog.V(1).Infof("registered route driver %q", name) + klog.V(1).Info(utils.FormatTunnel("registered route driver %q", name)) drivers[name] = factory } @@ -54,7 +55,7 @@ func New(name string, cfg *config.Config) (Driver, error) { driversMutex.Lock() defer driversMutex.Unlock() if _, found := drivers[name]; !found { - klog.Fatalf("route driver %q not found", name) + klog.Fatal(utils.FormatTunnel("route driver %q not found", name)) } return drivers[name](cfg) } diff --git a/pkg/networkengine/routedriver/vxlan/utils.go b/pkg/networkengine/routedriver/vxlan/utils.go index f8e7f32..f3abf9b 100644 --- a/pkg/networkengine/routedriver/vxlan/utils.go +++ b/pkg/networkengine/routedriver/vxlan/utils.go @@ -28,6 +28,7 @@ import ( "k8s.io/klog/v2" netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink" + "github.com/openyurtio/raven/pkg/utils" ) const ( @@ -41,7 +42,7 @@ func ensureVxlanLink(vxlan netlink.Vxlan, vtepIP net.IP) (netlink.Link, error) { return link } if _, ok := err.(netlink.LinkNotFoundError); !ok { - klog.Errorf("error get vxlan link: %v", err) + klog.Errorf(utils.FormatTunnel("error get vxlan link: %v", err)) } return nil } @@ -90,7 +91,6 @@ func ensureVxlanLink(vxlan netlink.Vxlan, vtepIP net.IP) (netlink.Link, error) { if err != nil { return nil, fmt.Errorf("error ensure filter: %v", err) } - return vxLink, nil } diff --git a/pkg/networkengine/routedriver/vxlan/vxlan.go b/pkg/networkengine/routedriver/vxlan/vxlan.go index 292093a..811029b 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan.go @@ -24,17 +24,18 @@ import ( "os" "syscall" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" "github.com/vdobler/ht/errorlist" "github.com/vishvananda/netlink" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" "github.com/openyurtio/raven/cmd/agent/app/config" "github.com/openyurtio/raven/pkg/networkengine/routedriver" networkutil "github.com/openyurtio/raven/pkg/networkengine/util" ipsetutil "github.com/openyurtio/raven/pkg/networkengine/util/ipset" iptablesutil "github.com/openyurtio/raven/pkg/networkengine/util/iptables" "github.com/openyurtio/raven/pkg/types" + "github.com/openyurtio/raven/pkg/utils" ) const ( @@ -73,11 +74,11 @@ type vxlan struct { func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error)) (err error) { if network.LocalEndpoint == nil || len(network.RemoteEndpoints) == 0 { - klog.Info("no local gateway or remote gateway is found, cleaning up route setting") + klog.Info(utils.FormatTunnel("no local gateway or remote gateway is found, cleaning up route setting")) return vx.Cleanup() } if len(network.LocalNodeInfo) == 1 { - klog.Infof("only gateway node exist in current gateway, cleaning up route setting") + klog.Infof(utils.FormatTunnel("only gateway node exist in current gateway, cleaning up route setting")) return vx.Cleanup() } @@ -211,7 +212,7 @@ func (vx *vxlan) MTU(network *types.Network) (int, error) { } // nodeInfo returns node info of the current node. -func (vx *vxlan) nodeInfo(network *types.Network) *v1alpha1.NodeInfo { +func (vx *vxlan) nodeInfo(network *types.Network) *v1beta1.NodeInfo { return network.LocalNodeInfo[vx.nodeName] } diff --git a/pkg/networkengine/routedriver/vxlan/vxlan_test.go b/pkg/networkengine/routedriver/vxlan/vxlan_test.go index 4ae370d..37878c0 100644 --- a/pkg/networkengine/routedriver/vxlan/vxlan_test.go +++ b/pkg/networkengine/routedriver/vxlan/vxlan_test.go @@ -20,10 +20,10 @@ import ( "net" "testing" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" networkutil "github.com/openyurtio/raven/pkg/networkengine/util" netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink" "github.com/openyurtio/raven/pkg/types" @@ -368,7 +368,7 @@ func TestVxlan_Apply(t *testing.T) { }, PrivateIP: "192.168.1.1", }, - LocalNodeInfo: map[types.NodeName]*v1alpha1.NodeInfo{ + LocalNodeInfo: map[types.NodeName]*v1beta1.NodeInfo{ "node-1": { NodeName: "node-1", PrivateIP: "192.168.1.1", @@ -402,7 +402,7 @@ func TestVxlan_Apply(t *testing.T) { PrivateIP: "192.168.1.3", }, }, - RemoteNodeInfo: map[types.NodeName]*v1alpha1.NodeInfo{ + RemoteNodeInfo: map[types.NodeName]*v1beta1.NodeInfo{ "node-2": { NodeName: "node-2", PrivateIP: "192.168.1.2", diff --git a/pkg/networkengine/vpndriver/driver.go b/pkg/networkengine/vpndriver/driver.go index f5133f4..f377025 100644 --- a/pkg/networkengine/vpndriver/driver.go +++ b/pkg/networkengine/vpndriver/driver.go @@ -15,7 +15,8 @@ import ( ) const ( - DefaultPSK string = "openyurt-raven" + DefaultPSK = "openyurt-raven" + DefaultVPNPort = "4500" ) // Driver is the interface for VPN implementation. diff --git a/pkg/networkengine/vpndriver/driver_test.go b/pkg/networkengine/vpndriver/driver_test.go index 4e65aa6..1ac9f10 100644 --- a/pkg/networkengine/vpndriver/driver_test.go +++ b/pkg/networkengine/vpndriver/driver_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" "github.com/openyurtio/raven/cmd/agent/app/config" "github.com/openyurtio/raven/pkg/types" @@ -103,9 +103,9 @@ func TestFindCentralGwFn(t *testing.T) { PrivateIP: "192.168.1.1", UnderNAT: false, }, - LocalNodeInfo: make(map[types.NodeName]*v1alpha1.NodeInfo), + LocalNodeInfo: make(map[types.NodeName]*v1beta1.NodeInfo), RemoteEndpoints: make(map[types.GatewayName]*types.Endpoint), - RemoteNodeInfo: make(map[types.NodeName]*v1alpha1.NodeInfo), + RemoteNodeInfo: make(map[types.NodeName]*v1beta1.NodeInfo), } tests := []struct { diff --git a/pkg/networkengine/vpndriver/libreswan/libreswan.go b/pkg/networkengine/vpndriver/libreswan/libreswan.go index 8f75219..8a4098a 100644 --- a/pkg/networkengine/vpndriver/libreswan/libreswan.go +++ b/pkg/networkengine/vpndriver/libreswan/libreswan.go @@ -31,6 +31,7 @@ import ( netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink" "github.com/openyurtio/raven/pkg/networkengine/vpndriver" "github.com/openyurtio/raven/pkg/types" + "github.com/openyurtio/raven/pkg/utils" ) const ( @@ -57,6 +58,7 @@ const ( type libreswan struct { connections map[string]*vpndriver.Connection nodeName types.NodeName + listenPort string iptables iptablesutil.IPTablesInterface } @@ -90,23 +92,24 @@ func New(cfg *config.Config) (vpndriver.Driver, error) { return &libreswan{ connections: make(map[string]*vpndriver.Connection), nodeName: types.NodeName(cfg.NodeName), + listenPort: cfg.Tunnel.VPNPort, }, nil } func (l *libreswan) Apply(network *types.Network, routeDriverMTUFn func(*types.Network) (int, error)) (err error) { errList := errorlist.List{} if network.LocalEndpoint == nil || len(network.RemoteEndpoints) == 0 { - klog.Info("no local gateway or remote gateway is found, cleaning vpn connections") + klog.Info(utils.FormatTunnel("no local gateway or remote gateway is found, cleaning vpn connections")) return l.Cleanup() } if network.LocalEndpoint.NodeName != l.nodeName { - klog.Infof("the current node is not gateway node, cleaning vpn connections") + klog.Info(utils.FormatTunnel("the current node is not gateway node, cleaning vpn connections")) return l.Cleanup() } desiredConnections := l.computeDesiredConnections(network) if len(desiredConnections) == 0 { - klog.Infof("no desired connections, cleaning vpn connections") + klog.Info(utils.FormatTunnel("no desired connections, cleaning vpn connections")) return l.Cleanup() } @@ -204,7 +207,7 @@ func (l *libreswan) whackConnectToEndpoint(connectionName string, connection *vp "--id", leftID, "--host", connection.LocalEndpoint.String(), "--client", connection.LocalSubnet, - "--ikeport", "4500", + "--ikeport", l.listenPort, ) } else { args = append(args, "--psk", "--encrypt", "--forceencaps", "--name", connectionName, @@ -219,7 +222,7 @@ func (l *libreswan) whackConnectToEndpoint(connectionName string, connection *vp "--id", rightID, "--host", connection.RemoteEndpoint.PublicIP, "--client", connection.RemoteSubnet, - "--ikeport", "4500") + "--ikeport", l.listenPort) } else { args = append(args, "--to", "--id", rightID, diff --git a/pkg/networkengine/vpndriver/wireguard/wireguard.go b/pkg/networkengine/vpndriver/wireguard/wireguard.go index d1ee22e..6c640ae 100644 --- a/pkg/networkengine/vpndriver/wireguard/wireguard.go +++ b/pkg/networkengine/vpndriver/wireguard/wireguard.go @@ -23,9 +23,10 @@ import ( "net" "os" "reflect" + "strconv" "time" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" "github.com/pkg/errors" "github.com/vdobler/ht/errorlist" "github.com/vishvananda/netlink" @@ -57,8 +58,8 @@ const ( // DeviceName specifies name of WireGuard network device. DeviceName = "raven-wg0" - // ListenPort specifies port of WireGuard listened. - ListenPort = 4500 + // DefaultListenPort specifies port of WireGuard listened. + DefaultListenPort = 4500 ) var findCentralGw = vpndriver.FindCentralGwFn @@ -80,13 +81,19 @@ type wireguard struct { iptables iptablesutil.IPTablesInterface nodeName types.NodeName ravenClient client.Client + listenPort int } func New(cfg *config.Config) (vpndriver.Driver, error) { + port, err := strconv.Atoi(cfg.Tunnel.VPNPort) + if err != nil { + port = DefaultListenPort + } return &wireguard{ connections: make(map[string]*vpndriver.Connection), nodeName: types.NodeName(cfg.NodeName), ravenClient: cfg.Manager.GetClient(), + listenPort: port, }, nil } @@ -127,7 +134,7 @@ func (w *wireguard) Init() error { func (w *wireguard) isWgDeviceChanged(existing, desired netlink.Link) bool { if d, err := w.wgClient.Device(DeviceName); err == nil { - if d.ListenPort == ListenPort && reflect.DeepEqual(d.PrivateKey, w.privateKey) { + if d.ListenPort == w.listenPort && reflect.DeepEqual(d.PrivateKey, w.privateKey) { return false } } @@ -183,7 +190,7 @@ func (w *wireguard) ensureWgLink(network *types.Network, routeDriverMTUFn func(* return fmt.Errorf("failed to add WireGuard device: %v", err) } - port := ListenPort + port := w.listenPort // Init Configure the device. peerConfigs := make([]wgtypes.PeerConfig, 0) cfg := wgtypes.Config{ @@ -298,7 +305,7 @@ func (w *wireguard) Apply(network *types.Network, routeDriverMTUFn func(*types.N allowedIPs = append(allowedIPs, parseSubnets(centralAllowedIPs)...) } - remotePort := ListenPort + remotePort := w.listenPort ka := KeepAliveInterval peerConfigs = append(peerConfigs, wgtypes.PeerConfig{ PublicKey: *newKey, @@ -478,7 +485,7 @@ func (w *wireguard) removePeer(key *wgtypes.Key) error { func (w *wireguard) configGatewayPublicKey(gwName string, nodeName string) error { err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // get localGateway from api server - var apiGw v1alpha1.Gateway + var apiGw v1beta1.Gateway err := w.ravenClient.Get(context.Background(), client.ObjectKey{ Name: gwName, }, &apiGw) @@ -486,7 +493,7 @@ func (w *wireguard) configGatewayPublicKey(gwName string, nodeName string) error return err } for k, v := range apiGw.Spec.Endpoints { - if v.NodeName == nodeName { + if v.NodeName == nodeName && v.Type == v1beta1.Tunnel { if apiGw.Spec.Endpoints[k].Config == nil { apiGw.Spec.Endpoints[k].Config = make(map[string]string) } diff --git a/pkg/proxyengine/proxyclient/proxyclient.go b/pkg/proxyengine/proxyclient/proxyclient.go new file mode 100644 index 0000000..cbf73e8 --- /dev/null +++ b/pkg/proxyengine/proxyclient/proxyclient.go @@ -0,0 +1,123 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyclient + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + certificatesv1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + anp "sigs.k8s.io/apiserver-network-proxy/pkg/agent" + + "github.com/openyurtio/openyurt/pkg/util/certmanager" + "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" + "github.com/openyurtio/raven/pkg/proxyengine" + "github.com/openyurtio/raven/pkg/utils" +) + +type ProxyClient struct { + name string + ip string + certDir string + metaAddress string + servers map[string]*tls.Config + client kubernetes.Interface +} + +func NewProxyClient(proxyCfg *proxyengine.EnginConfig, addresses []string, kubeCfg *rest.Config) (*ProxyClient, error) { + if len(addresses) < 1 { + return nil, fmt.Errorf("failed to get proxy server address") + } + client, err := kubernetes.NewForConfig(kubeCfg) + if err != nil { + return nil, err + } + servers := make(map[string]*tls.Config) + for _, addr := range addresses { + servers[addr] = nil + } + return &ProxyClient{name: proxyCfg.Name, ip: proxyCfg.IP, certDir: proxyCfg.CertDir, metaAddress: proxyCfg.MetaAddress, client: client, servers: servers}, nil +} + +func (c *ProxyClient) Start(ctx context.Context) error { + certMgrCfg := &factory.CertManagerConfig{ + ComponentName: utils.RavenProxyClientName, + CertDir: c.certDir, + SignerName: certificatesv1.KubeAPIServerClientSignerName, + CommonName: utils.RavenProxyClientCSRCN, + Organizations: []string{utils.RavenCSROrg}, + DNSNames: []string{c.name}, + IPs: []net.IP{net.ParseIP(c.ip)}, + } + clientCertManager, err := factory.NewCertManagerFactory(c.client).New(certMgrCfg) + if err != nil { + klog.Errorf(utils.FormatProxyClient("failed to new cert manager factory for proxy client %s, error %s", c.name, err.Error())) + return fmt.Errorf("failed to new cert manager factory for proxy client %s, error %s", c.name, err.Error()) + } + clientCertManager.Start() + _ = wait.PollUntil(5*time.Second, func() (bool, error) { + if clientCertManager.Current() != nil { + return true, nil + } + klog.Infof(utils.FormatProxyClient("certificate %s not signed, waiting...", certMgrCfg.CommonName)) + return false, nil + }, ctx.Done()) + for addr := range c.servers { + tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCA(clientCertManager, addr, utils.RavenCAFile) + if err != nil { + klog.Error(utils.FormatProxyClient("failed to generate TLS Config")) + return fmt.Errorf("failed to generate TLS Config") + } + c.servers[addr] = tlsCfg + } + klog.Infof("certificate %s ok", certMgrCfg.CommonName) + + utils.RunMetaServer(ctx, c.metaAddress) + c.run(ctx.Done()) + return nil +} + +func (c *ProxyClient) run(stopCh <-chan struct{}) { + for addr, cert := range c.servers { + client := c.NewClient(addr, cert, stopCh) + client.Serve() + klog.Infof(utils.FormatProxyClient("start serving grpc request redirected from %s", addr)) + } +} + +func (c *ProxyClient) NewClient(dstAddr string, tlsCfg *tls.Config, stopCh <-chan struct{}) *anp.ClientSet { + dialOption := grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)) + cc := &anp.ClientSetConfig{ + AgentID: c.name, + Address: dstAddr, + AgentIdentifiers: fmt.Sprintf("host=%s", c.name), + SyncInterval: 5 * time.Second, + ProbeInterval: 5 * time.Second, + DialOptions: []grpc.DialOption{dialOption}, + ServiceAccountTokenPath: "", + } + return cc.NewAgentClientSet(stopCh) +} diff --git a/pkg/proxyengine/proxyserver/interceptor.go b/pkg/proxyengine/proxyserver/interceptor.go new file mode 100644 index 0000000..ad988a3 --- /dev/null +++ b/pkg/proxyengine/proxyserver/interceptor.go @@ -0,0 +1,324 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyserver + +import ( + "bufio" + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/apiserver/pkg/util/flushwriter" + "k8s.io/apiserver/pkg/util/wsstream" + "k8s.io/klog/v2" + + "github.com/openyurtio/raven/pkg/utils" +) + +var ( + SupportedHeaders = []string{utils.RavenProxyHostHeaderKey, utils.RavenProxyUserHeaderKey} + IOReaderPool sync.Pool +) + +const HTTPCloseErr = "use of closed network connection" + +// newBufioReader retrieves a cached Reader from the pool if the pool is not empty, +// otherwise creates a new one +func newBufioReader(r io.Reader) *bufio.Reader { + if v := IOReaderPool.Get(); v != nil { + br := v.(*bufio.Reader) + br.Reset(r) + return br + } + return bufio.NewReader(r) +} + +// putBufioReader puts the Reader to the pool. +func putBufioReader(br *bufio.Reader) { + br.Reset(nil) + IOReaderPool.Put(br) +} + +type Connect interface { + Connect(r *http.Request) (net.Conn, error) +} + +type Interceptor struct { + dial func(address string, header http.Header, isTLS bool) (net.Conn, error) +} + +func NewInterceptor(udsFile string, cfg *tls.Config) http.Handler { + if cfg == nil { + return nil + } + dial := func(address string, header http.Header, isTLS bool) (net.Conn, error) { + var conn net.Conn + var err error + switch header.Get(utils.RavenProxyServerForwardModeHeaderKey) { + case utils.RavenProxyServerForwardLocalMode: + if address == "" { + return nil, fmt.Errorf("failed to connect, address is empty") + } + conn, err = net.Dial("tcp", address) + if err != nil { + return nil, fmt.Errorf("failed to dial server %s, error %s", address, err.Error()) + } + + case utils.RavenProxyServerForwardRemoteMode: + if udsFile == "" { + return nil, fmt.Errorf("failed to connect, uds file is empty") + } + conn, err = net.Dial("unix", udsFile) + if err != nil { + return nil, fmt.Errorf("failed to dial proxy %s, error %s", udsFile, err.Error()) + } + var connectHeaders string + for _, val := range SupportedHeaders { + if v := header.Get(val); len(v) != 0 { + connectHeaders = fmt.Sprintf("%s\r\n%s:%s", connectHeaders, val, v) + } + } + _, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: localhost%s\r\n\r\n", address, connectHeaders) + if err != nil { + return nil, fmt.Errorf("failed format connect header, error %s", err.Error()) + } + br := newBufioReader(conn) + defer putBufioReader(br) + resp, err := http.ReadResponse(br, nil) + if err != nil { + conn.Close() + return nil, fmt.Errorf("reading HTTP response from CONNECT to %s failed %s", address, err.Error()) + } + if resp.StatusCode != 200 { + conn.Close() + return nil, fmt.Errorf("proxy err while dialing %s, code %d: %v", address, resp.StatusCode, resp.Status) + } + default: + return nil, fmt.Errorf("unrecognize the proxy forwarding mode") + } + if isTLS { + cfg.InsecureSkipVerify = false + tlsConn := tls.Client(conn, cfg) + if err := tlsConn.Handshake(); err != nil { + conn.Close() + return nil, fmt.Errorf("fail to setup TLS handshake to %s: error %s", address, err.Error()) + } + return tlsConn, nil + } + return conn, nil + } + return &Interceptor{dial: dial} +} + +func (c *Interceptor) Connect(r *http.Request) (net.Conn, error) { + return c.dial(r.Host, r.Header, r.TLS != nil) +} + +func (c *Interceptor) ServeHTTP(w http.ResponseWriter, r *http.Request) { + conn, err := c.Connect(r) + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "failed to setup the proxy for %s, error %s", r.Host, err) + return + } + defer conn.Close() + + err = r.Write(conn) + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "failed to write request to conn, err %s", err.Error()) + return + } + + if httpstream.IsUpgradeRequest(r) { + serveUpgradeRequest(conn, w, r) + } else { + serveRequest(conn, w, r) + } +} + +// serverRequest serves the normal requests, e.g., kubectl logs +func serveRequest(conn net.Conn, w http.ResponseWriter, r *http.Request) { + klog.Info(utils.FormatProxyServer("interceptor: start serving request %s with header: host %s, proxy mode: %s", + r.URL.String(), r.Header[utils.RavenProxyHostHeaderKey], r.Header[utils.RavenProxyServerForwardModeHeaderKey])) + defer klog.Info(utils.FormatProxyServer("interceptor: stop serving request %s with header: host %s, proxy mode: %s", + r.URL.String(), r.Header[utils.RavenProxyHostHeaderKey], r.Header[utils.RavenProxyServerForwardModeHeaderKey])) + br := newBufioReader(conn) + defer putBufioReader(br) + resp, err := http.ReadResponse(br, r) + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "fail to read response from the conn: %s", err.Error()) + return + } + defer resp.Body.Close() + + if wsstream.IsWebSocketRequest(r) { + wsReader := wsstream.NewReader(resp.Body, true, wsstream.NewDefaultReaderProtocols()) + if err = wsReader.Copy(w, r); err != nil { + klog.ErrorS(err, "error encountered while streaming results via websocket") + } + return + } + + copyHeader(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + writer := w.(io.Writer) + if isChunked(resp) { + stopCh := make(chan struct{}) + defer close(stopCh) + go func(r *http.Request, conn net.Conn, stopCh chan struct{}) { + ctx := r.Context() + select { + case <-stopCh: + klog.Info(utils.FormatProxyServer("chunked request(%s) normally exit", r.URL.String())) + case <-ctx.Done(): + klog.Info(utils.FormatProxyServer("chunked request(%s) to agent(%s) closed by cloud client, %v", r.URL.String(), + r.Header.Get(utils.RavenProxyHostHeaderKey), ctx.Err())) + conn.Close() + } + }(r, conn, stopCh) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + writer = flushwriter.Wrap(w) + } + _, err = io.Copy(writer, resp.Body) + if err != nil && !isHTTPCloseError(err) { + klog.ErrorS(err, utils.FormatProxyServer("failed to copy response from proxy server to the frontend")) + } +} + +func serveUpgradeRequest(conn net.Conn, w http.ResponseWriter, r *http.Request) { + klog.Info(utils.FormatProxyServer("interceptor: start serving streaming request %s with header: host %s, proxy mode: %s", + r.URL.String(), r.Header[utils.RavenProxyHostHeaderKey], r.Header[utils.RavenProxyServerForwardModeHeaderKey])) + defer klog.Info(utils.FormatProxyServer("interceptor: stop serving streaming request %s with header: host %s, proxy mode: %s", + r.URL.String(), r.Header[utils.RavenProxyHostHeaderKey], r.Header[utils.RavenProxyServerForwardModeHeaderKey])) + + resp, rawResp, err := getResponse(conn) + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "proxy connection error %s", err.Error()) + return + } + hijacker, ok := w.(http.Hijacker) + if !ok { + logAndHTTPError(w, http.StatusServiceUnavailable, "can't assert response to http.Hijacker") + return + } + frontend, _, err := hijacker.Hijack() + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "fail to hijack response: %s", err.Error()) + return + } + defer frontend.Close() + + if resp.StatusCode != http.StatusSwitchingProtocols { + deadline := time.Now().Add(10 * time.Second) + if err = conn.SetReadDeadline(deadline); err != nil { + klog.Errorf(utils.FormatProxyServer("failed set proxy connect deadline, error %s", err.Error())) + } + if err = frontend.SetReadDeadline(deadline); err != nil { + klog.Errorf(utils.FormatProxyServer("failed set frontend connect deadline, error %s", err.Error())) + } + err = resp.Write(frontend) + if err != nil && !isHTTPCloseError(err) { + klog.Errorf(utils.FormatProxyServer("error proxying un-upgrade response from proxy channel to frontend: %s", err.Error())) + } + return + } + + if len(rawResp) > 0 { + if _, err = frontend.Write(rawResp); err != nil { + klog.Errorf(utils.FormatProxyServer("error proxying response bytes from tunnel to client: %s", err.Error())) + } + } + + readerComplete, writerComplete := make(chan struct{}), make(chan struct{}) + go func() { + _, err = io.Copy(conn, frontend) + if err != nil && !isHTTPCloseError(err) { + klog.Errorf(utils.FormatProxyServer("error proxying data from frontend to proxy channel: %s", err.Error())) + } + close(writerComplete) + }() + + go func() { + _, err = io.Copy(frontend, conn) + if err != nil && !isHTTPCloseError(err) { + klog.Errorf(utils.FormatProxyServer("error proxying data from proxy channel to frontend: %s", err.Error())) + } + close(readerComplete) + }() + + select { + case <-writerComplete: + case <-readerComplete: + } +} + +func logAndHTTPError(w http.ResponseWriter, errCode int, format string, i ...interface{}) { + errMsg := fmt.Sprintf(format, i...) + klog.Error(utils.FormatProxyServer(errMsg)) + http.Error(w, errMsg, errCode) +} + +func copyHeader(dst, src http.Header) { + for key, val := range src { + for _, v := range val { + dst.Add(key, v) + } + } +} + +// isChunked verify the specified response is chunked stream or not. +func isChunked(resp *http.Response) bool { + for _, h := range resp.Header[http.CanonicalHeaderKey("Transfer-Encoding")] { + if strings.Contains(strings.ToLower(h), strings.ToLower("chunked")) { + return true + } + } + + for _, te := range resp.TransferEncoding { + if strings.Contains(strings.ToLower(te), strings.ToLower("chunked")) { + return true + } + } + return false +} + +func isHTTPCloseError(err error) bool { + return strings.Contains(strings.ToLower(err.Error()), HTTPCloseErr) +} + +// getResponseCode reads a http response from the given reader, returns the response, +// the bytes read from the reader, and any error encountered +func getResponse(r io.Reader) (*http.Response, []byte, error) { + rawResponse := bytes.NewBuffer(make([]byte, 0, 256)) + // Save the bytes read while reading the response headers into the rawResponse buffer + br := newBufioReader(io.TeeReader(r, rawResponse)) + defer putBufioReader(br) + resp, err := http.ReadResponse(br, nil) + if err != nil { + return nil, nil, err + } + // return the http response and the raw bytes consumed from the reader in the process + return resp, rawResponse.Bytes(), nil +} diff --git a/pkg/proxyengine/proxyserver/manageheader.go b/pkg/proxyengine/proxyserver/manageheader.go new file mode 100644 index 0000000..e07e4e0 --- /dev/null +++ b/pkg/proxyengine/proxyserver/manageheader.go @@ -0,0 +1,245 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyserver + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "time" + + "math/rand" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/raven" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/raven/pkg/metrics" + "github.com/openyurtio/raven/pkg/utils" +) + +var ( + requestsPathPrefix = map[string]struct{}{ + "exec": {}, + "attach": {}, + "portForward": {}, + "containerLogs": {}} +) + +type WrapperHandler interface { + Handler(handler http.Handler) http.Handler +} + +type headerManger struct { + client client.Client + gatewayName string + isIPv4 bool +} + +func NewHeaderManager(client client.Client, gatewayName string, isIPv4 bool) WrapperHandler { + return &headerManger{ + client: client, + gatewayName: gatewayName, + isIPv4: isIPv4, + } +} + +func (h *headerManger) Handler(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r == nil { + klog.Errorf(utils.FormatProxyServer("request is nil, skip it")) + return + } + oldHost := r.Host + klog.Info(utils.FormatProxyServer("request with host %s and url %s is processed by header manager", oldHost, r.URL.String())) + var host, ip, port string + var err error + if isAPIServerRequest(r) { + host, ip, port, err = h.getAPIServerRequestDestAddress(r) + if err != nil { + logAndHTTPError(w, http.StatusBadRequest, "request host %s and url %s is invalid, %s", r.Host, r.URL.String(), err.Error()) + return + } + } else { + host, ip, port, err = h.getNormalRequestDestAddress(r) + if err != nil { + logAndHTTPError(w, http.StatusBadRequest, "request host %s and url %s is invalid, %s", r.Host, r.URL.String(), err.Error()) + return + } + } + if host == "" || ip == "" || port == "" { + logAndHTTPError(w, http.StatusBadRequest, "request host %s and url %s is invalid, %s", r.Host, r.URL.String(), err.Error()) + return + } + // Complete request header information + if r.TLS == nil { + r.URL.Scheme = "http" + } else { + r.URL.Scheme = "https" + } + proxyDest := fmt.Sprintf("%s:%s", ip, port) + proxyHost := fmt.Sprintf("%s:%s", host, port) + proxyMode, err := h.getProxyMode(host) + if err != nil { + logAndHTTPError(w, http.StatusServiceUnavailable, "request host %s and url %s can not get proxy mode, error %s", + r.Host, r.URL.String(), err.Error()) + return + } + + r.Host = proxyDest + r.Header.Set("Host", proxyDest) + r.URL.Host = proxyDest + r.Header.Set(utils.RavenProxyHostHeaderKey, proxyHost) + r.Header.Set(utils.RavenProxyDestHeaderKey, proxyDest) + r.Header.Set(utils.RavenProxyServerForwardModeHeaderKey, proxyMode) + // observe metrics + metrics.Metrics.IncInFlightRequests(r.Method, r.URL.Path) + defer metrics.Metrics.DecInFlightRequests(r.Method, r.URL.Path) + + klog.Infoln(utils.FormatProxyServer("start handling request %s %s, req.Host changed from %s to %s, remote address is %s", + r.Method, r.URL.String(), oldHost, r.Host, r.RemoteAddr)) + start := time.Now() + handler.ServeHTTP(w, r) + klog.Infoln(utils.FormatProxyServer("finish handle request %s %s, handle lasts %v", r.Method, r.URL.String(), time.Since(start))) + }) +} + +func (h *headerManger) getAPIServerRequestDestAddress(r *http.Request) (name, ip, port string, err error) { + nodeName := r.Header.Get(utils.RavenProxyHostHeaderKey) + if nodeName == "" { + parts := strings.Split(r.URL.Path, "/") + var pod v1.Pod + err = h.client.Get(context.TODO(), client.ObjectKey{Namespace: parts[2], Name: parts[3]}, &pod) + if err != nil { + return "", "", "", err + } + if pod.Spec.NodeName != "" { + nodeName = pod.Spec.NodeName + } + } + var node v1.Node + err = h.client.Get(context.TODO(), client.ObjectKey{Name: nodeName}, &node) + if err != nil { + return "", "", "", err + } + + name, err = h.getGatewayNodeName(&node) + if err != nil { + return "", "", "", fmt.Errorf("gateway include node %s, has no active endpoints, error %s", + node.Name, err.Error()) + } + + ip = getNodeIP(&node) + if ip == "" { + return "", "", "", fmt.Errorf("node %s ip is empty", node.Name) + } + port = strconv.Itoa(int(node.Status.DaemonEndpoints.KubeletEndpoint.Port)) + return name, ip, port, nil +} + +func (h *headerManger) getNormalRequestDestAddress(r *http.Request) (name, ip, port string, err error) { + var nodeName string + nodeName, port, err = net.SplitHostPort(r.Host) + if err != nil { + return "", "", "", err + } + if nodeName == "" { + nodeName = r.Header.Get(utils.RavenProxyHostHeaderKey) + } + ipAddress := net.ParseIP(nodeName) + if ipAddress != nil { + klog.Warning(utils.FormatProxyServer("raven proxy server not support request.Host is %s", r.Host)) + return "", "", "", nil + } + var node v1.Node + err = h.client.Get(context.TODO(), client.ObjectKey{Name: nodeName}, &node) + if err != nil { + return "", "", "", err + } + name, err = h.getGatewayNodeName(&node) + if err != nil { + return "", "", "", fmt.Errorf("gateway include node %s, has no active endpoints, error %s", + node.Name, err.Error()) + } + ip = getNodeIP(&node) + if ip == "" { + return "", "", "", fmt.Errorf("node %s ip is empty", node.Name) + } + return name, ip, port, nil +} + +func (h *headerManger) getProxyMode(nodeName string) (string, error) { + var gw v1beta1.Gateway + err := h.client.Get(context.TODO(), types.NamespacedName{Name: h.gatewayName}, &gw) + if err != nil { + return "", err + } + for _, localNode := range gw.Status.Nodes { + if localNode.NodeName == nodeName { + return utils.RavenProxyServerForwardLocalMode, nil + } + } + return utils.RavenProxyServerForwardRemoteMode, nil +} + +func isAPIServerRequest(r *http.Request) bool { + parts := strings.Split(r.URL.Path, "/") + if len(parts) < 5 { + return false + } + if _, ok := requestsPathPrefix[parts[1]]; ok { + return true + } + return false +} + +func getNodeIP(node *v1.Node) string { + var ip string + if node != nil && node.Status.Addresses != nil { + for _, nodeAddr := range node.Status.Addresses { + if nodeAddr.Type == v1.NodeInternalIP { + ip = nodeAddr.Address + break + } + } + } + return ip +} + +func (h *headerManger) getGatewayNodeName(node *v1.Node) (string, error) { + gwName, ok := node.Labels[raven.LabelCurrentGateway] + if !ok { + return node.Name, nil + } + var gw v1beta1.Gateway + err := h.client.Get(context.TODO(), types.NamespacedName{Name: gwName}, &gw) + if err != nil { + if apierrors.IsNotFound(err) { + return node.Name, nil + } + return "", err + } + rand.Seed(time.Now().Unix()) + return gw.Status.ActiveEndpoints[rand.Intn(len(gw.Status.ActiveEndpoints))].NodeName, nil +} diff --git a/pkg/proxyengine/proxyserver/proxyserver.go b/pkg/proxyengine/proxyserver/proxyserver.go new file mode 100644 index 0000000..6359851 --- /dev/null +++ b/pkg/proxyengine/proxyserver/proxyserver.go @@ -0,0 +1,281 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyserver + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "strings" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" + anpserver "sigs.k8s.io/apiserver-network-proxy/pkg/server" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/raven" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/util/certmanager" + "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" + ravenutils "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" + "github.com/openyurtio/raven/pkg/proxyengine" + "github.com/openyurtio/raven/pkg/utils" +) + +type ProxyServer struct { + nodeName string + nodeIP string + metaAddress string + exposedAddress string + internalInsecureAddress string + internalSecureAddress string + certDir string + interceptorUDSFile string + certDNSNames []string + gateway *v1beta1.Gateway + certIPs []net.IP + clientSet kubernetes.Interface + client client.Client + rootCert *x509.CertPool + serverTLSConfig *tls.Config + proxyTLSConfig *tls.Config +} + +func NewProxyServer(cfg *proxyengine.EnginConfig, client client.Client, kubeCfg *rest.Config, gw *v1beta1.Gateway) (*ProxyServer, error) { + certIPs := make([]net.IP, 0) + if cfg.CertIPs != "" { + for _, ip := range strings.Split(cfg.CertIPs, ",") { + if addr := net.ParseIP(ip); addr != nil { + certIPs = append(certIPs, addr) + } + } + } + certDNSNames := make([]string, 0) + if cfg.CertDNSNames != "" { + for _, dns := range strings.Split(cfg.CertDNSNames, ",") { + if dns != "" { + certDNSNames = append(certDNSNames, dns) + } + } + } + server := &ProxyServer{ + nodeName: cfg.Name, + gateway: gw, + nodeIP: cfg.IP, + metaAddress: cfg.MetaAddress, + internalInsecureAddress: cfg.InternalInsecureAddress, + internalSecureAddress: cfg.InternalSecureAddress, + exposedAddress: cfg.ExposedAddress, + interceptorUDSFile: cfg.InterceptorUDSFile, + certDir: cfg.CertDir, + certDNSNames: certDNSNames, + certIPs: certIPs, + client: client, + } + var err error + server.clientSet, err = kubernetes.NewForConfig(kubeCfg) + if err != nil { + return nil, err + } + server.rootCert, err = certmanager.GenCertPoolUseCA(utils.RavenCAFile) + if err != nil { + return nil, err + } + return server, nil +} + +func (c *ProxyServer) Start(ctx context.Context) error { + dnsNames, IPs := c.getProxyServerIPsAndDNSName(ctx) + certFactory := factory.NewCertManagerFactory(c.clientSet) + serverCertCfg := &factory.CertManagerConfig{ + IPs: append(c.certIPs, IPs...), + IPGetter: func() ([]net.IP, error) { + _, ips := c.getProxyServerIPsAndDNSName(ctx) + return ips, nil + }, + DNSNames: append(c.certDNSNames, dnsNames...), + ComponentName: utils.RavenProxyServerName, + CertDir: c.certDir, + SignerName: certificatesv1.KubeletServingSignerName, + CommonName: fmt.Sprintf("system:node:%s", utils.RavenProxyServerCSRCN), + Organizations: []string{user.NodesGroup}, + ForServerUsage: true, + } + serverCertMgr, err := certFactory.New(serverCertCfg) + if err != nil { + klog.Errorf(utils.FormatProxyServer("failed to new server cert manager factory for proxy server %s, error %s", c.nodeName, err.Error())) + return fmt.Errorf("failed to new server cert manager factory for proxy server %s, error %s", c.nodeName, err.Error()) + } + serverCertMgr.Start() + + proxyCertCfg := &factory.CertManagerConfig{ + CertDir: c.certDir, + ComponentName: utils.RavenProxyUserName, + SignerName: certificatesv1.KubeAPIServerClientSignerName, + Organizations: []string{utils.RavenCSROrg}, + CommonName: utils.RavenProxyUserCSRCN, + } + proxyCertMgr, err := certFactory.New(proxyCertCfg) + if err != nil { + klog.Errorf(utils.FormatProxyServer("failed to new proxy cert manager factory for proxy server %s, error %s", c.nodeName, err.Error())) + return fmt.Errorf("failed to new proxy cert manager factory for proxy server %s, error %s", c.nodeName, err.Error()) + } + proxyCertMgr.Start() + + _ = wait.PollUntil(5*time.Second, func() (bool, error) { + if serverCertMgr.Current() != nil && proxyCertMgr.Current() != nil { + return true, nil + } + klog.Infof(utils.FormatProxyServer("certificate %s and %s not signed, waiting...", serverCertCfg.ComponentName, proxyCertCfg.ComponentName)) + return false, nil + }, ctx.Done()) + + klog.Infof(utils.FormatProxyServer("certificate %s and %s ok", serverCertCfg.ComponentName, proxyCertCfg.ComponentName)) + c.serverTLSConfig, err = certmanager.GenTLSConfigUseCurrentCertAndCertPool(serverCertMgr.Current, c.rootCert, "server") + if err != nil { + return err + } + c.proxyTLSConfig, err = certmanager.GenTLSConfigUseCurrentCertAndCertPool(proxyCertMgr.Current, c.rootCert, "client") + if err != nil { + return err + } + utils.RunMetaServer(ctx, c.metaAddress) + c.runServers(ctx) + return nil +} + +func (c *ProxyServer) runServers(ctx context.Context) { + klog.Info(utils.FormatProxyServer("start proxy server")) + strategy := []anpserver.ProxyStrategy{anpserver.ProxyStrategyDestHost} + proxyServer := anpserver.NewProxyServer(c.nodeName, strategy, 1, &anpserver.AgentTokenAuthenticationOptions{}) + NewProxies(&anpserver.Tunnel{Server: proxyServer}, c.interceptorUDSFile).Run(ctx) + interceptor := NewInterceptor(c.interceptorUDSFile, c.proxyTLSConfig) + headerMgr := NewHeaderManager(c.client, c.gateway.GetName(), utilnet.IsIPv4String(c.nodeIP)) + NewMaster(headerMgr.Handler(interceptor), c.serverTLSConfig, c.internalSecureAddress, c.internalInsecureAddress).Run(ctx) + NewAgent(c.serverTLSConfig, proxyServer, c.exposedAddress).Run(ctx) +} + +func (c *ProxyServer) getProxyServerIPsAndDNSName(ctx context.Context) (dnsName []string, ipAddr []net.IP) { + ipAddr = append(ipAddr, net.ParseIP(c.nodeIP)) + ipAddr = append(ipAddr, net.ParseIP(utils.DefaultLoopBackIP4)) + dnsName = append(dnsName, c.nodeName) + _ = wait.PollImmediateWithContext(ctx, 10*time.Second, 10*time.Minute, func(ctx context.Context) (done bool, err error) { + var svc v1.Service + err = c.client.Get(ctx, types.NamespacedName{Namespace: ravenutils.WorkingNamespace, Name: ravenutils.GatewayProxyInternalService}, &svc) + if err != nil { + klog.Errorf(utils.FormatProxyServer("failed to get internal service %s/%s to get proxy server IPs and DNSNames, error %s", + svc.GetNamespace(), svc.GetName(), err.Error())) + return false, nil + } + dnsName = append(dnsName, getDefaultDomainsForSvc(svc.GetNamespace(), svc.GetName())...) + if svc.Spec.ClusterIP != "" { + ipAddr = append(ipAddr, net.ParseIP(svc.Spec.ClusterIP)) + } + var svcList v1.ServiceList + err = c.client.List(ctx, &svcList, &client.ListOptions{ + LabelSelector: labels.Set{ + raven.LabelCurrentGateway: c.gateway.GetName(), + raven.LabelCurrentGatewayType: v1beta1.Proxy, + ravenutils.LabelCurrentGatewayEndpoints: c.nodeName, + }.AsSelector(), + }) + if err != nil { + klog.Errorf(utils.FormatProxyServer("failed to get public service for gateway %s, node %s to get proxy server IPs and DNSNames, error %s", + c.gateway.GetName(), c.nodeName, err.Error())) + return false, nil + } + + for _, svc = range svcList.Items { + dnsName = append(dnsName, getDefaultDomainsForSvc(svc.GetNamespace(), svc.GetName())...) + dnsName = append(dnsName, getExternalDNSName(&svc)...) + ipAddr = append(ipAddr, getExternalIPForSvc(&svc)...) + if svc.Spec.ClusterIP != "" { + ipAddr = append(ipAddr, net.ParseIP(svc.Spec.ClusterIP)) + } + if svc.Status.LoadBalancer.Ingress != nil { + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + ipAddr = append(ipAddr, net.ParseIP(ing.IP)) + } + if ing.Hostname != "" { + dnsName = append(dnsName, ing.Hostname) + } + } + } + } + return true, nil + }) + return +} + +func getExternalIPForSvc(svc *v1.Service) []net.IP { + ret := make([]net.IP, 0) + if svc.Annotations == nil { + return ret + } + _, ok := svc.Annotations["raven.openyurt.io/public-service-external-ip"] + if !ok { + return ret + } + addresses := strings.Split(svc.Annotations["raven.openyurt.io/public-service-external-ip"], ",") + for _, val := range addresses { + ip := net.ParseIP(strings.TrimSpace(val)) + if ip != nil { + ret = append(ret, ip) + } + } + return ret +} + +func getExternalDNSName(svc *v1.Service) []string { + ret := make([]string, 0) + if svc.Annotations == nil { + return ret + } + _, ok := svc.Annotations["raven.openyurt.io/public-service-external-dns-name"] + if !ok { + return ret + } + dnsNames := strings.Split(svc.Annotations["raven.openyurt.io/public-service-external-dns-name"], ",") + for _, val := range dnsNames { + ret = append(ret, strings.TrimSpace(val)) + } + return ret +} + +func getDefaultDomainsForSvc(ns, name string) []string { + domains := make([]string, 0) + if len(ns) == 0 || len(name) == 0 { + return domains + } + domains = append(domains, name) + domains = append(domains, fmt.Sprintf("%s.%s", name, ns)) + domains = append(domains, fmt.Sprintf("%s.%s.svc", name, ns)) + domains = append(domains, fmt.Sprintf("%s.%s.svc.cluster.local", name, ns)) + return domains +} diff --git a/pkg/proxyengine/proxyserver/servers.go b/pkg/proxyengine/proxyserver/servers.go new file mode 100644 index 0000000..52c41e4 --- /dev/null +++ b/pkg/proxyengine/proxyserver/servers.go @@ -0,0 +1,166 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyserver + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "time" + + "github.com/openyurtio/raven/pkg/utils" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "k8s.io/klog/v2" + anpserver "sigs.k8s.io/apiserver-network-proxy/pkg/server" + anpagent "sigs.k8s.io/apiserver-network-proxy/proto/agent" +) + +type Server interface { + Run(ctx context.Context) +} + +type proxies struct { + handler http.Handler + udsSockFile string +} + +func NewProxies(handler http.Handler, udsFile string) Server { + return &proxies{handler: handler, udsSockFile: udsFile} +} + +func (p *proxies) Run(ctx context.Context) { + go func(ctx context.Context) { + klog.Info(utils.FormatProxyServer("start listen unix %s", p.udsSockFile)) + defer klog.Info(utils.FormatProxyServer("finish listen unix %s", p.udsSockFile)) + server := &http.Server{ + Handler: p.handler, + ReadTimeout: 10 * time.Second, + } + listen, err := net.Listen("unix", p.udsSockFile) + if err != nil { + klog.Errorf("proxies failed to listen uds %s", err.Error()) + } + defer listen.Close() + go func(ctx context.Context) { + <-ctx.Done() + err = server.Shutdown(ctx) + if err != nil { + klog.Errorf("failed to shutdown proxies server, error %s", err.Error()) + } + }(ctx) + if err := server.Serve(listen); err != nil { + klog.Errorf("proxies failed to serving request through uds %s", err.Error()) + } + + }(ctx) +} + +type master struct { + handler http.Handler + secureAddr string + insecureAddr string + tlsCfg *tls.Config +} + +func NewMaster(handler http.Handler, tlsCfg *tls.Config, secureAddr, insecureAddr string) Server { + return &master{handler: handler, tlsCfg: tlsCfg, secureAddr: secureAddr, insecureAddr: insecureAddr} +} + +func (m *master) Run(ctx context.Context) { + go func(ctx context.Context) { + klog.Info(utils.FormatProxyServer("start handling https request from master at %s", m.secureAddr)) + defer klog.Info(utils.FormatProxyServer("finish handling https request from master at %s", m.secureAddr)) + server := http.Server{ + Addr: m.secureAddr, + Handler: m.handler, + ReadTimeout: 10 * time.Second, + TLSConfig: m.tlsCfg, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + go func(ctx context.Context) { + <-ctx.Done() + err := server.Shutdown(ctx) + if err != nil { + klog.Errorf("failed to shutdown master secure server, error %s", err.Error()) + } + }(ctx) + if err := server.ListenAndServeTLS("", ""); err != nil { + klog.Errorf(utils.FormatProxyServer("failed to serve https request from master: %s", err.Error())) + } + }(ctx) + + go func(ctx context.Context) { + klog.Infof(utils.FormatProxyServer("start handling https request from master at %s", m.insecureAddr)) + defer klog.Infof(utils.FormatProxyServer("finish handling https request from master at %s", m.insecureAddr)) + server := http.Server{ + Addr: m.insecureAddr, + Handler: m.handler, + ReadTimeout: 10 * time.Second, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + go func(ctx context.Context) { + <-ctx.Done() + err := server.Shutdown(ctx) + if err != nil { + klog.Errorf("failed to shutdown master insecure server, error %s", err.Error()) + } + }(ctx) + if err := server.ListenAndServe(); err != nil { + klog.Errorf(utils.FormatProxyServer("failed to serve https request from master: %s", err.Error())) + } + }(ctx) +} + +type agent struct { + address string + tlsCfg *tls.Config + proxyServer *anpserver.ProxyServer +} + +func NewAgent(tlsCfg *tls.Config, proxyServer *anpserver.ProxyServer, address string) Server { + return &agent{tlsCfg: tlsCfg, proxyServer: proxyServer, address: address} +} + +func (c *agent) Run(ctx context.Context) { + go func(ctx context.Context) { + klog.Info(utils.FormatProxyServer("start handling grpc request from proxy client at %s", c.address)) + defer klog.Info(utils.FormatProxyServer("finish handling grpc request from proxy client at %s", c.address)) + ka := keepalive.ServerParameters{ + MaxConnectionIdle: 10 * time.Minute, + Time: 10 * time.Second, + Timeout: 5 * time.Second, + } + grpcServer := grpc.NewServer(grpc.KeepaliveParams(ka), grpc.Creds(credentials.NewTLS(c.tlsCfg))) + anpagent.RegisterAgentServiceServer(grpcServer, c.proxyServer) + listen, err := net.Listen("tcp", c.address) + if err != nil { + klog.Errorf(utils.FormatProxyServer("failed to listen to agent on %s: %s", c.address, err.Error())) + return + } + defer listen.Close() + go func(ctx context.Context) { + <-ctx.Done() + grpcServer.Stop() + }(ctx) + if err := grpcServer.Serve(listen); err != nil { + klog.Errorf(utils.FormatProxyServer("failed to server grpc request from proxy agent server, error %s", err.Error())) + } + }(ctx) +} diff --git a/pkg/proxyengine/utils.go b/pkg/proxyengine/utils.go new file mode 100644 index 0000000..666d0e4 --- /dev/null +++ b/pkg/proxyengine/utils.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxyengine + +type EnginConfig struct { + Name string + IP string + CertDir string + MetaAddress string + + //proxy server + GatewayName string + CertDNSNames string + CertIPs string + InterceptorUDSFile string + ExposedAddress string + InternalInsecureAddress string + InternalSecureAddress string +} diff --git a/pkg/tunnelengine/tunnelagent.go b/pkg/tunnelengine/tunnelagent.go new file mode 100644 index 0000000..ab87d17 --- /dev/null +++ b/pkg/tunnelengine/tunnelagent.go @@ -0,0 +1,287 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnelengine + +import ( + "context" + "fmt" + "net" + "reflect" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/EvilSuperstars/go-cidrman" + "github.com/openyurtio/openyurt/pkg/apis/raven" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + ravenutils "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" + "github.com/openyurtio/raven/pkg/networkengine/routedriver" + "github.com/openyurtio/raven/pkg/networkengine/vpndriver" + "github.com/openyurtio/raven/pkg/types" + "github.com/openyurtio/raven/pkg/utils" +) + +type TunnelHandler struct { + nodeName string + forwardNodeIP bool + ownGateway *v1beta1.Gateway + + ravenClient client.Client + routeDriver routedriver.Driver + vpnDriver vpndriver.Driver + + nodeInfos map[types.NodeName]*v1beta1.NodeInfo + network *types.Network + lastSeenNetwork *types.Network +} + +func NewTunnelHandler(nodeName string, forwardNodeIP bool, client client.Client, routeDriver routedriver.Driver, vpnDriver vpndriver.Driver) *TunnelHandler { + return &TunnelHandler{ + nodeName: nodeName, + forwardNodeIP: forwardNodeIP, + ravenClient: client, + routeDriver: routeDriver, + vpnDriver: vpnDriver, + } +} + +// sync syncs full state according to the gateway list. +func (c *TunnelHandler) Handler() error { + var gws v1beta1.GatewayList + err := c.ravenClient.List(context.Background(), &gws) + if err != nil { + return err + } + // As we are going to rebuild a full state, so cleanup before proceeding. + c.network = &types.Network{ + LocalEndpoint: nil, + RemoteEndpoints: make(map[types.GatewayName]*types.Endpoint), + LocalNodeInfo: make(map[types.NodeName]*v1beta1.NodeInfo), + RemoteNodeInfo: make(map[types.NodeName]*v1beta1.NodeInfo), + } + c.nodeInfos = make(map[types.NodeName]*v1beta1.NodeInfo) + + c.ownGateway, err = utils.GetOwnGateway(c.ravenClient, c.nodeName) + if err != nil { + return fmt.Errorf("failed to get own gateway, error %s", err.Error()) + } + + for i := range gws.Items { + // try to update public IP if empty. + gw := &gws.Items[i] + if ep := getTunnelActiveEndpoints(gw); ep != nil && ep.PublicIP == "" { + err := c.configGatewayPublicIP(gw) + if err != nil { + klog.ErrorS(err, "error config gateway public ip", "gateway", klog.KObj(gw)) + } + continue + } + if !c.shouldHandleGateway(gw) { + continue + } + c.syncNodeInfo(gw.Status.Nodes) + } + for i := range gws.Items { + gw := &gws.Items[i] + if !c.shouldHandleGateway(gw) { + continue + } + c.syncGateway(gw) + } + if reflect.DeepEqual(c.network, c.lastSeenNetwork) { + klog.Info("network not changed, skip to process") + return nil + } + nw := c.network.Copy() + klog.InfoS("applying network", "localEndpoint", nw.LocalEndpoint, "remoteEndpoint", nw.RemoteEndpoints) + err = c.vpnDriver.Apply(nw, c.routeDriver.MTU) + if err != nil { + return err + } + err = c.routeDriver.Apply(nw, c.vpnDriver.MTU) + if err != nil { + return err + } + + // Only update lastSeenNetwork when all operations succeeded. + c.lastSeenNetwork = c.network + return nil +} + +func (c *TunnelHandler) syncNodeInfo(nodes []v1beta1.NodeInfo) { + for _, v := range nodes { + c.nodeInfos[types.NodeName(v.NodeName)] = v.DeepCopy() + } +} + +func (c *TunnelHandler) appendNodeIP(gw *v1beta1.Gateway) { + for i := range gw.Status.Nodes { + nodeSubnet := net.IPNet{ + IP: net.ParseIP(gw.Status.Nodes[i].PrivateIP), + Mask: []byte{0xff, 0xff, 0xff, 0xff}, + } + gw.Status.Nodes[i].Subnets = append(gw.Status.Nodes[i].Subnets, nodeSubnet.String()) + } +} + +func (c *TunnelHandler) getMergedSubnets(nodeInfo []v1beta1.NodeInfo) []string { + subnets := make([]string, 0) + for _, n := range nodeInfo { + subnets = append(subnets, n.Subnets...) + } + subnets, _ = cidrman.MergeCIDRs(subnets) + return subnets +} + +func (c *TunnelHandler) syncGateway(gw *v1beta1.Gateway) { + if c.forwardNodeIP { + c.appendNodeIP(gw) + } + aep := getTunnelActiveEndpoints(gw) + subnets := c.getMergedSubnets(gw.Status.Nodes) + cfg := make(map[string]string) + for k := range aep.Config { + cfg[k] = aep.Config[k] + } + var nodeInfo *v1beta1.NodeInfo + if nodeInfo = c.nodeInfos[types.NodeName(aep.NodeName)]; nodeInfo == nil { + klog.Errorf("node %s is found in Endpoint but not existed in NodeInfo", aep.NodeName) + return + } + ep := &types.Endpoint{ + GatewayName: types.GatewayName(gw.Name), + NodeName: types.NodeName(aep.NodeName), + Subnets: subnets, + PrivateIP: nodeInfo.PrivateIP, + PublicIP: aep.PublicIP, + UnderNAT: aep.UnderNAT, + Config: cfg, + } + var isLocalGateway bool + defer func() { + for _, v := range gw.Status.Nodes { + if isLocalGateway { + c.network.LocalNodeInfo[types.NodeName(v.NodeName)] = v.DeepCopy() + } else { + c.network.RemoteNodeInfo[types.NodeName(v.NodeName)] = v.DeepCopy() + } + } + }() + + if gw.Name == c.ownGateway.Name { + c.network.LocalEndpoint = ep + isLocalGateway = true + } else { + c.network.RemoteEndpoints[types.GatewayName(gw.Name)] = ep + } + +} + +func (c *TunnelHandler) shouldHandleGateway(gateway *v1beta1.Gateway) bool { + if getTunnelActiveEndpoints(gateway) == nil { + klog.InfoS("no active endpoint , waiting for sync", "gateway", klog.KObj(gateway)) + return false + } + if getTunnelActiveEndpoints(gateway).PublicIP == "" { + klog.InfoS("no public IP for gateway, waiting for sync", "gateway", klog.KObj(gateway)) + return false + } + if c.ownGateway == nil { + klog.InfoS(fmt.Sprintf("no own gateway for node %s, skip it", c.nodeName), "gateway", klog.KObj(gateway)) + return false + } + return true +} + +func (c *TunnelHandler) configGatewayPublicIP(gateway *v1beta1.Gateway) error { + if getTunnelActiveEndpoints(gateway).NodeName != c.nodeName { + return nil + } + var publicIP string + var err error + if gateway.Spec.ExposeType == v1beta1.ExposeTypeLoadBalancer { + publicIP, err = c.getLoadBalancerPublicIP(gateway.GetName()) + if err != nil { + return err + } + } else { + publicIP, err = utils.GetPublicIP() + if err != nil { + return err + } + } + + // retry to update public ip of localGateway + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + // get localGateway from api server + var apiGw v1beta1.Gateway + err := c.ravenClient.Get(context.Background(), client.ObjectKey{ + Name: gateway.Name, + }, &apiGw) + if err != nil { + return err + } + for k, v := range apiGw.Spec.Endpoints { + if v.NodeName == c.nodeName { + apiGw.Spec.Endpoints[k].PublicIP = publicIP + err = c.ravenClient.Update(context.Background(), &apiGw) + return err + } + } + return nil + }) + return err +} + +func (c *TunnelHandler) getLoadBalancerPublicIP(gwName string) (string, error) { + var svcList v1.ServiceList + err := c.ravenClient.List(context.TODO(), &svcList, &client.ListOptions{ + LabelSelector: labels.Set{ + raven.LabelCurrentGateway: gwName, + raven.LabelCurrentGatewayType: v1beta1.Tunnel, + ravenutils.LabelCurrentGatewayEndpoints: c.nodeName, + }.AsSelector(), + }) + if err != nil { + return "", err + } + if len(svcList.Items) == 0 { + return "", apierrors.NewNotFound(v1.Resource("service"), fmt.Sprintf("%s-%s", "x-raven-proxy-svc-%s", gwName)) + } + svc := svcList.Items[0] + if svc.Status.LoadBalancer.Ingress == nil && len(svc.Status.LoadBalancer.Ingress) == 0 { + return "", apierrors.NewNotFound(v1.Resource("service"), svc.GetName()) + } + publicIP := svc.Status.LoadBalancer.Ingress[0].IP + if publicIP == "" { + return "", apierrors.NewServiceUnavailable(fmt.Sprintf("service %s/%s has no public ingress", svc.GetNamespace(), svc.GetName())) + } + return publicIP, nil +} + +func getTunnelActiveEndpoints(gw *v1beta1.Gateway) *v1beta1.Endpoint { + for _, aep := range gw.Status.ActiveEndpoints { + if aep.Type == v1beta1.Tunnel { + return aep.DeepCopy() + } + } + return nil +} diff --git a/pkg/types/network.go b/pkg/types/network.go index 753de80..0c53ea6 100644 --- a/pkg/types/network.go +++ b/pkg/types/network.go @@ -17,7 +17,7 @@ package types import ( - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" ) // GatewayName is the type representing the name of Gateway. @@ -64,12 +64,12 @@ type Network struct { // Equals to nil if there is no local gateway node. LocalEndpoint *Endpoint // LocalNodeInfo stores NodeInfo of all nodes in local gateway. - LocalNodeInfo map[NodeName]*v1alpha1.NodeInfo + LocalNodeInfo map[NodeName]*v1beta1.NodeInfo // RemoteEndpoints is the Endpoint of all remote gateway nodes, indexed by their gateway name. // Equals to nil if there is no remote gateway node. RemoteEndpoints map[GatewayName]*Endpoint // RemoteNodeInfo stores NodeInfo of all nodes in remote gateways - RemoteNodeInfo map[NodeName]*v1alpha1.NodeInfo + RemoteNodeInfo map[NodeName]*v1beta1.NodeInfo } func (n *Network) Copy() *Network { @@ -78,9 +78,9 @@ func (n *Network) Copy() *Network { } nw := &Network{ LocalEndpoint: n.LocalEndpoint.Copy(), - LocalNodeInfo: make(map[NodeName]*v1alpha1.NodeInfo), + LocalNodeInfo: make(map[NodeName]*v1beta1.NodeInfo), RemoteEndpoints: make(map[GatewayName]*Endpoint), - RemoteNodeInfo: make(map[NodeName]*v1alpha1.NodeInfo), + RemoteNodeInfo: make(map[NodeName]*v1beta1.NodeInfo), } for k, v := range n.RemoteEndpoints { nw.RemoteEndpoints[k] = v.Copy() diff --git a/pkg/types/network_test.go b/pkg/types/network_test.go index f0fc583..9efc71a 100644 --- a/pkg/types/network_test.go +++ b/pkg/types/network_test.go @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" ) const ( @@ -102,9 +102,9 @@ func TestNCopy(t *testing.T) { LocalEndpoint: &Endpoint{ PrivateIP: "192.168.1.1", }, - LocalNodeInfo: make(map[NodeName]*v1alpha1.NodeInfo), + LocalNodeInfo: make(map[NodeName]*v1beta1.NodeInfo), RemoteEndpoints: make(map[GatewayName]*Endpoint), - RemoteNodeInfo: make(map[NodeName]*v1alpha1.NodeInfo), + RemoteNodeInfo: make(map[NodeName]*v1beta1.NodeInfo), } tests := []struct { diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go new file mode 100644 index 0000000..a7b5d0e --- /dev/null +++ b/pkg/utils/constants.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +const ( + MaxRetries = 15 + RavenProxyClientName = "raven-proxy-client" + RavenProxyServerName = "raven-proxy-server" + RavenProxyUserName = "raven-proxy-user" + + RavenProxyClientCSRCN = "tunnel-agent-client" + RavenProxyServerCSRCN = "tunnel-proxy-server" + RavenProxyUserCSRCN = "tunnel-proxy-client" + RavenCSROrg = "openyurt:yurttunnel" + + RavenCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + RavenTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + RavenProxyServerCertDir = "/var/lib/raven/proxy.server/pki" + RavenProxyClientCertDir = "/var/lib/raven/proxy.client/pki" + RavenProxyServerUDSFile = "/tmp/interceptor-proxier.sock" + + DefaultLoopBackIP4 = "127.0.0.1" + DefaultLoopBackIP6 = "::1" + RavenProxyHostHeaderKey = "X-Tunnel-Proxy-Host" + RavenProxyDestHeaderKey = "X-Tunnel-Proxy-Dest" + RavenProxyUserHeaderKey = "User-Agent" + RavenProxyServerForwardModeHeaderKey = "Forward-Mode" + RavenProxyServerForwardLocalMode = "Local" + RavenProxyServerForwardRemoteMode = "Remote" + + WorkingNamespace = "kube-system" + RavenConfigName = "raven-cfg" +) diff --git a/pkg/utils/metaserver.go b/pkg/utils/metaserver.go new file mode 100644 index 0000000..b5cfb22 --- /dev/null +++ b/pkg/utils/metaserver.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "net/http" + + "k8s.io/klog/v2" + + "github.com/gorilla/mux" + "github.com/openyurtio/openyurt/pkg/util/profile" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func RunMetaServer(ctx context.Context, addr string) { + go func(ctx context.Context) { + klog.Infoln("start handling meta requests(metrics/pprof)", "server endpoint", addr) + defer klog.Infoln("meta server stopped", "server endpoint", addr) + muxHandler := mux.NewRouter() + muxHandler.Handle("/metrics", promhttp.Handler()) + // register handler for pprof + profile.Install(muxHandler) + metaServer := &http.Server{ + Addr: addr, + Handler: muxHandler, + MaxHeaderBytes: 1 << 20, + } + go func(ctx context.Context) { + <-ctx.Done() + err := metaServer.Shutdown(ctx) + if err != nil { + klog.Errorf("failed to shutdown meta server, error %s", err.Error()) + } + }(ctx) + err := metaServer.ListenAndServe() + if err != nil { + klog.ErrorS(err, "meta server could not listen") + } + }(ctx) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..ed4b427 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func FormatProxyServer(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("ProxyServer: %s", s) +} + +func FormatProxyClient(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("ProxyClient: %s", s) +} + +func FormatTunnel(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("Tunnel: %s", s) +} + +func FormatRavenEngine(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("RavenEngine: %s", s) +} + +func GetOwnGateway(client client.Client, nodeName string) (*v1beta1.Gateway, error) { + var gwList v1beta1.GatewayList + err := client.List(context.TODO(), &gwList) + if err != nil { + return nil, err + } + for _, gw := range gwList.Items { + for _, node := range gw.Status.Nodes { + if node.NodeName == nodeName { + return gw.DeepCopy(), nil + } + } + } + return nil, nil +}