diff --git a/Makefile b/Makefile index 4650c7e1..e0de3507 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ clangformat: @echo -clang-format -i -style=file internal/bpf/*.c internal/bpf/*.h -clang-format -i -style=file internal/bpf/xdp-pass/*.c + -clang-format -i -style=file internal/bpf/xdp-afxdp-redirect/*.c @echo @echo @@ -37,18 +38,22 @@ format: clangformat @echo @echo +buildxdp: + @echo "****** Build xdp_pass ******" + make -C ./internal/bpf/xdp-pass/ + @echo "****** Build xdp_afxdp_redirect ******" + make -C ./internal/bpf/xdp-afxdp-redirect/ + @echo + buildc: @echo "****** Build BPF ******" @echo gcc ./internal/bpf/bpfWrapper.c -lxdp -c -o ./internal/bpf/bpfWrapper.o ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o &> /dev/null - @echo "****** Build xdp_pass ******" - make -C ./internal/bpf/xdp-pass/ - @echo @echo @echo -builddp: buildc +builddp: buildc buildxdp @echo "****** Build DP ******" @echo go build -o ./bin/afxdp-dp ./cmd/deviceplugin diff --git a/README.md b/README.md index 2c55e0f5..a6a19113 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,10 @@ Note: User 0 does not imply that the pod needs to be privileged. UdsServerDisable is a Boolean configuration. If set to true, devices in this pool will not have the BPF app loaded onto the netdev. This means no UDS server is spun up when a device is allocated to a pod. By default, this is set to false. +#### BpfMapPinningEnable + +BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration. + #### UdsTimeout UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds. diff --git a/cmd/deviceplugin/main.go b/cmd/deviceplugin/main.go index 289a21c2..167d8ef6 100644 --- a/cmd/deviceplugin/main.go +++ b/cmd/deviceplugin/main.go @@ -25,6 +25,7 @@ import ( "github.com/intel/afxdp-plugins-for-kubernetes/constants" "github.com/intel/afxdp-plugins-for-kubernetes/internal/deviceplugin" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver" "github.com/intel/afxdp-plugins-for-kubernetes/internal/host" "github.com/intel/afxdp-plugins-for-kubernetes/internal/logformats" "github.com/intel/afxdp-plugins-for-kubernetes/internal/networking" @@ -97,9 +98,16 @@ func main() { } logging.Infof("Host meets requirements") + //START THE SYNCER SERVER TODO CHECK BPF MAP + dpCniSyncerServer, err := dpcnisyncerserver.NewSyncerServer() + if err != nil { + logging.Errorf("Error creating the DpCniSyncerServer") + } + logging.Debugf("DP<=>CNI grpc Syncer started") + // pool configs logging.Infof("Getting device pools") - poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler) + poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler, dpCniSyncerServer) if err != nil { logging.Warningf("Error getting device pools: %v", err) exit(constants.Plugins.DevicePlugin.ExitPoolError) diff --git a/constants/constants.go b/constants/constants.go index d03fea6b..3ae67af0 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -82,6 +82,10 @@ var ( udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point udsPodPath = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod + /* BPF*/ + bpfMapPodPath = "/tmp/xsks_map" + xsk_map = "/xsks_map" + udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets /* Handshake*/ @@ -129,6 +133,8 @@ var ( DeviceFile deviceFile /* DeviceFile contains constants related to the devicefile */ EthtoolFilter ethtoolFilter + /* Bpf contains constants related to the BPF Map pinning */ + Bpf bpf ) type cni struct { @@ -213,6 +219,11 @@ type uds struct { Handshake handshake } +type bpf struct { + BpfMapPodPath string + Xsk_map string +} + type handshake struct { Version string RequestVersion string @@ -334,6 +345,11 @@ func init() { }, } + Bpf = bpf{ + BpfMapPodPath: bpfMapPodPath, + Xsk_map: xsk_map, + } + EthtoolFilter = ethtoolFilter{ EthtoolFilterRegex: ethtoolFilterRegex, } diff --git a/deployments/daemonset-kind.yaml b/deployments/daemonset-kind.yaml index 19936ee3..292d2022 100644 --- a/deployments/daemonset-kind.yaml +++ b/deployments/daemonset-kind.yaml @@ -13,7 +13,8 @@ data: { "name":"myPool", "mode":"primary", - "UdsServerDisable": false, + "UdsServerDisable": true, + "BpfMapPinningEnable": true, "drivers":[ { "name":"veth" @@ -72,6 +73,8 @@ spec: volumeMounts: - name: unixsock mountPath: /tmp/afxdp_dp/ + - name: bpfmappinning + mountPath: /var/run/afxdp_dp/ mountPropagation: Bidirectional - name: devicesock mountPath: /var/lib/kubelet/device-plugins/ @@ -87,6 +90,9 @@ spec: - name: unixsock hostPath: path: /tmp/afxdp_dp/ + - name: bpfmappinning + hostPath: + path: /var/run/afxdp_dp/ - name: devicesock hostPath: path: /var/lib/kubelet/device-plugins/ diff --git a/examples/cndp-0-0.yaml b/examples/cndp-0-0.yaml new file mode 100644 index 00000000..8080306a --- /dev/null +++ b/examples/cndp-0-0.yaml @@ -0,0 +1,26 @@ +# A working example of BPF MAP PINNING +apiVersion: v1 +kind: Pod +metadata: + name: cndp-0-0 + annotations: + k8s.v1.cni.cncf.io/networks: afxdp-network +spec: + containers: + - name: cndp-0 + command: ["/bin/bash"] + args: ["-c", "./jsonc_gen.sh -kp ; cndpfwd -c config.jsonc lb;"] + image: quay.io/mtahhan/cndp-map-pinning:latest + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + #capabilities: + #add: + # - NET_RAW + # - IPC_LOCK + # - BPF + resources: + requests: + afxdp/myPool: '1' + limits: + afxdp/myPool: '1' diff --git a/examples/nad_with_syncer.yaml b/examples/nad_with_syncer.yaml new file mode 100644 index 00000000..a8cfce99 --- /dev/null +++ b/examples/nad_with_syncer.yaml @@ -0,0 +1,27 @@ +# WARNING: This is an example definition only. Remove all comments before use. + +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: afxdp-network # Name of this network, pods will request this network by name + annotations: + k8s.v1.cni.cncf.io/resourceName: afxdp/myPool # Needs to match the device plugin pool name / resource type +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "afxdp", # CNI binary, leave as afxdp + "mode": "primary", # CNI mode setting (required) + "logFile": "afxdp-cni.log", # CNI log file location (optional) + "logLevel": "debug", # CNI logging level (optional) + "dpSyncer": true, # Sync with Device Plugin over gRPC MUST BE SET for bpf map pinning + "ipam": { # CNI IPAM plugin and associated config (optional) + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.220", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.1.1" + } + }' diff --git a/go.mod b/go.mod index f66ec954..e5f808b3 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,12 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.1.1 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 + github.com/golang/protobuf v1.5.3 github.com/google/gofuzz v1.1.0 github.com/google/uuid v1.3.0 github.com/intel/afxdp-plugins-for-kubernetes/pkg/goclient v0.0.0 github.com/intel/afxdp-plugins-for-kubernetes/pkg/subfunctions v0.0.0 + github.com/moby/sys/mount v0.3.3 github.com/pkg/errors v0.9.1 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 github.com/sirupsen/logrus v1.9.0 @@ -17,6 +19,7 @@ require ( github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 golang.org/x/net v0.17.0 google.golang.org/grpc v1.53.0 + google.golang.org/protobuf v1.30.0 // indirect gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.25.2 k8s.io/kubelet v0.25.2 diff --git a/go.sum b/go.sum index 9ea88d1f..267bd2b5 100644 --- a/go.sum +++ b/go.sum @@ -735,8 +735,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -894,8 +895,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= +github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= @@ -1774,8 +1779,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/images/amd64.dockerfile b/images/amd64.dockerfile index 10b683ec..4e299231 100644 --- a/images/amd64.dockerfile +++ b/images/amd64.dockerfile @@ -17,8 +17,8 @@ COPY . /usr/src/afxdp_k8s_plugins WORKDIR /usr/src/afxdp_k8s_plugins RUN apt-get update \ && apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 \ -&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6 \ -&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 \ +&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.7~deb12u1 \ +&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.7~deb12u1 \ && apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 \ && make buildcni @@ -39,4 +39,5 @@ COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-pass/xdp_pass.o /afxdp/xdp_pass.o +COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.o /afxdp/xdp_afxdp_redirect.o ENTRYPOINT ["/afxdp/entrypoint.sh"] diff --git a/internal/bpf/bpfWrapper.c b/internal/bpf/bpfWrapper.c index c427a28a..87b67087 100644 --- a/internal/bpf/bpfWrapper.c +++ b/internal/bpf/bpfWrapper.c @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,8 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include +#include #include // for XDP_FLAGS_DRV_MODE #include // for if_nametoindex +#include +#include #include #include // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd @@ -183,8 +188,13 @@ int Load_attach_bpf_xdp_pass(char *ifname) { if (!ifindex) { Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname); return -1; - } else { - Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex); + } + Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex); + + if (access(filename, O_RDONLY) < 0) { + Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename, + strerror(errno)); + return err; } Log_Info("%s: starting setup of xdp-pass program on " @@ -214,3 +224,52 @@ int Load_attach_bpf_xdp_pass(char *ifname) { return 0; } + +int Load_bpf_pin_xsk_map(char *ifname, char *pin_path) { + struct bpf_object *obj; + struct bpf_program *prog; + struct bpf_link *link; + int ifindex, map_fd = -1; + int err; + const char *prog_name = "xdp_afxdp_redirect"; + char *filename = "/afxdp/xdp_afxdp_redirect.o"; + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, bpf_opts, .pin_root_path = pin_path); + + ifindex = if_nametoindex(ifname); + if (!ifindex) { + Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname); + return -1; + } + Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex); + + if (access(filename, O_RDONLY) < 0) { + Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename, + strerror(errno)); + return err; + } + + Log_Info("%s: starting setup of xdp-redirect program on " + "interface %s (%d)", + __FUNCTION__, ifname, ifindex); + + /* Load the BPF program */ + prog = xdp_program__open_file(filename, NULL, NULL); + err = libxdp_get_error(prog); + if (err) { + libxdp_strerror(err, "Couldn’t load XDP program", + sizeof("Couldn’t load XDP program")); + Log_Error("%s: Couldn’t load XDP program\n", __FUNCTION__, filename); + return err; + } + + /* Attach the program to the interface at the xdp hook */ + err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0); + if (err) { + libxdp_strerror(err, "Couldn't attach the xdp pass program", + sizeof("Couldn't attach the xdp pass program")); + Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname); + return err; + } + + return 0; +} diff --git a/internal/bpf/bpfWrapper.go b/internal/bpf/bpfWrapper.go index ad491943..110cc9dd 100755 --- a/internal/bpf/bpfWrapper.go +++ b/internal/bpf/bpfWrapper.go @@ -39,6 +39,7 @@ type Handler interface { LoadBpfSendXskMap(ifname string) (int, error) LoadAttachBpfXdpPass(ifname string) error ConfigureBusyPoll(fd int, busyTimeout int, busyBudget int) error + LoadBpfPinXskMap(ifname, pin_path string) error Cleanbpf(ifname string) error } @@ -80,6 +81,19 @@ func (r *handler) LoadAttachBpfXdpPass(ifname string) error { return nil } +/* +LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_send_xsk_map +*/ +func (r *handler) LoadBpfPinXskMap(ifname, pin_path string) error { + err := int(C.Load_bpf_pin_xsk_map(C.CString(ifname), C.CString(pin_path))) + + if err < 0 { + return errors.New("error loading BPF program onto interface") + } + + return nil +} + /* ConfigureBusyPoll is the GoLang wrapper for the C function Configure_busy_poll */ diff --git a/internal/bpf/bpfWrapper.h b/internal/bpf/bpfWrapper.h index bf41107d..dbe181f7 100644 --- a/internal/bpf/bpfWrapper.h +++ b/internal/bpf/bpfWrapper.h @@ -19,6 +19,7 @@ int Load_bpf_send_xsk_map(char *ifname); int Load_attach_bpf_xdp_pass(char *ifname); +int Load_bpf_pin_xsk_map(char *ifname, char *pin_path); int Configure_busy_poll(int fd, int busy_timeout, int busy_budget); int Clean_bpf(char *ifname); diff --git a/internal/bpf/bpfWrapper_fake.go b/internal/bpf/bpfWrapper_fake.go index de42e295..17968ea5 100644 --- a/internal/bpf/bpfWrapper_fake.go +++ b/internal/bpf/bpfWrapper_fake.go @@ -45,6 +45,14 @@ func (f *fakeHandler) LoadAttachBpfXdpPass(ifname string) error { return nil } +/* +LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_pin_xsk_map +In this fakeHandler it does nothing. +*/ +func (f *fakeHandler) LoadBpfPinXskMap(ifname, pin_path string) error { + return nil +} + /* ConfigureBusyPoll is the GoLang wrapper for the C function Configure_busy_poll In this fakeHandler it does nothing. diff --git a/internal/bpf/mapManager.go b/internal/bpf/mapManager.go new file mode 100644 index 00000000..b505cbfb --- /dev/null +++ b/internal/bpf/mapManager.go @@ -0,0 +1,296 @@ +/* + * Copyright(c) Red Hat + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bpf + +import ( + "fmt" + "os" + "syscall" + + "github.com/google/uuid" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/host" + "github.com/moby/sys/mount" + "github.com/pkg/errors" + logging "github.com/sirupsen/logrus" +) + +const ( + pinnedMapBaseDir = "/var/run/afxdp_dp/" + pinnedMapDirFileMode = os.FileMode(0755) + bpffsDirFileMode = os.FileMode(0755) +) + +/* +MapManager is the interface defining the MAP MANAGER. +Implementations of this interface are the main type of this MapManager package. TODO UPDATE +*/ +type MapManager interface { + CreateBPFFS(dev, path string) (string, error) + DeleteBPFFS(dev string) error + AddMap(dev, path string) + GetMaps() (map[string]string, error) + GetBPFFS(dev string) (string, error) + GetName() string +} + +type PoolBpfMapManager struct { + Manager MapManager + Path string +} + +/* +MapManagerFactory is the interface defining a factory that creates and returns MapManagers. +Each device plugin poolManager will have its own MapManagerFactory and each time a +container is created the factory will create a MapManager to serve the +associated pinned BPF Map. TODO UPDATE THIS.... +*/ +type MapManagerFactory interface { + CreateMapManager(poolName, user string) (MapManager, string, error) +} + +/* +server implements the Server interface. It is the main type for this package. +*/ +type mapManager struct { + name string + maps map[string]string + bpffsPath string + uid string +} + +/* +mapManager implements the MapManager interface. +*/ +type mapManagerFactory struct { + MapManagerFactory +} + +/* +NewMapMangerFactory returns an implementation of the MapManagerFactory interface. +*/ +func NewMapMangerFactory() MapManagerFactory { + return &mapManagerFactory{} +} + +/* +CreateMapManager creates, initialises, and returns an implementation of the MapManager interface. +It also returns the filepath for bpf maps to be pinned. +*/ +func (f *mapManagerFactory) CreateMapManager(poolName, user string) (MapManager, string, error) { + + logging.Debugf(" CreateMapManager ") + if poolName == "" || user == "" { + return nil, "", errors.New("Error poolname or user not set") + } + p, err := createBPFFSBaseDirectory(poolName, user) + if err != nil { + return nil, "", errors.Wrapf(err, "Error creating BPFFS base directory %v", err.Error()) + } + logging.Infof("Created BPFFS Base directory %s", p) + + manager := &mapManager{ + maps: make(map[string]string), + bpffsPath: p, + uid: user, + name: poolName, + } + + return manager, p, nil +} + +func giveBpffsBasePermissions(path, user string) error { + if user != "0" { + logging.Infof("Giving permissions to UID %s", user) + err := host.GivePermissions(path, user, "rwx") + if err != nil { + return errors.Wrapf(err, "Error giving permissions to BPFFS path %s", err.Error()) + } + logging.Infof("User %s has access to %s", user, path) + } + return nil +} + +func createBPFFSBaseDirectory(p, user string) (string, error) { + + logging.Infof("Creating BPFFS Base directory %s", p) + + path := pinnedMapBaseDir + p + if _, err := os.Stat(path); os.IsNotExist(err) { + //create base directory if it not exists, with correct file permissions + if err = os.MkdirAll(path, pinnedMapDirFileMode); err != nil { + return "", errors.Wrapf(err, "Error creating BPFFS base directory %s: %v", pinnedMapBaseDir, err.Error()) + } + + if err = giveBpffsBasePermissions(path, user); err != nil { + return "", errors.Wrapf(err, "Error creating BPFFS base directory %s: %v", pinnedMapBaseDir, err.Error()) + } + } + + logging.Infof("Created Base BPFFS directory %s", path) + return path, nil +} + +func (m mapManager) CreateBPFFS(device, path string) (string, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", errors.Wrapf(err, "Error creating BPFFS mount point base directory %s doesn't exist: %v", pinnedMapBaseDir, err.Error()) + } + + bpffsPath, err := generateRandomBpffsName(m.bpffsPath) + if err != nil { + return "", errors.Wrapf(err, "Error generating BPFFS path: %s: %v", pinnedMapBaseDir, err.Error()) + } + + if err = os.MkdirAll(bpffsPath, bpffsDirFileMode); err != nil { + return "", errors.Wrapf(err, "Error creating BPFFS base directory %s: %v", pinnedMapBaseDir, err.Error()) + } + + if err = giveBpffsBasePermissions(bpffsPath, m.uid); err != nil { + return "", errors.Wrapf(err, "Error creating BPFFS base directory %s: %v", pinnedMapBaseDir, err.Error()) + } + logging.Infof("created a directory %s", bpffsPath) + + if err = syscall.Mount(bpffsPath, bpffsPath, "bpf", 0, ""); err != nil { + return "", errors.Wrapf(err, "failed to mount %s: %v", bpffsPath, err.Error()) + } + logging.Infof("Created BPFFS mount point at %s", bpffsPath) + + if err = mount.MakeShared(bpffsPath); err != nil { + return "", errors.Wrapf(err, "failed to make the BPFFS %s Shared: %v", bpffsPath, err.Error()) + } + + return bpffsPath, nil +} + +/* +generateRandomBpffsName will take the file directory path, and apply a unique name per each +bpffs created. +*/ +func generateRandomBpffsName(directory string) (string, error) { + + if _, err := os.Stat(directory); os.IsNotExist(err) { + return "", errors.Wrapf(err, "Error couldn't find directory %s: %v", directory, err.Error()) + } + + //get directory info + fileInfo, err := os.Stat(directory) + if err != nil { + logging.Errorf("Error getting directory info %s: %v", directory, err) + return "", err + } + + //verify it is a directory + if !fileInfo.IsDir() { + err = fmt.Errorf("%s is not a directory", directory) + logging.Errorf(err.Error()) + return "", err + } + + //verify the permissions are correct, in case of pre existing dir + if fileInfo.Mode().Perm() != bpffsDirFileMode { + err = fmt.Errorf("incorrect permissions on directory %s", directory) + logging.Errorf(err.Error()) + return "", err + } + + var bpffspath string + var count int = 0 + for { + if count >= 5 { + err = fmt.Errorf("error generating a unique UDS filepath") + logging.Errorf(err.Error()) + return "", err + } + + bpffsName, err := uuid.NewRandom() + if err != nil { + logging.Errorf("Error generating random UDS filename: %v", err) + } + + bpffspath = directory + bpffsName.String() + if _, err := os.Stat(bpffspath); os.IsNotExist(err) { + break + } + + logging.Debugf("%s already exists. Regenerating.", bpffspath) + count++ + } + + return bpffspath, nil +} + +/* +AddMap appends a netdev and its associated pinned xsk_map to the MapManager map of Maps. +*/ +func (m *mapManager) AddMap(dev, path string) { + m.maps[dev] = path +} + +/* +GetName +*/ +func (m *mapManager) GetName() string { + return m.name +} + +/* +GetMaps +*/ +func (m *mapManager) GetMaps() (map[string]string, error) { + + if m.maps != nil { + return nil, errors.New("No Maps found") + } + return m.maps, nil +} + +/* +GetBPFFS +*/ +func (m *mapManager) GetBPFFS(dev string) (string, error) { + + _, err := m.GetMaps() + if err != nil { + if p, ok := m.maps[dev]; ok { + return p, nil + } + } + + return "", errors.Wrapf(err, "Couldn't find any maps for dev %s: %v", dev, err.Error()) +} + +func (m *mapManager) DeleteBPFFS(dev string) error { + + bpffs, err := m.GetBPFFS(dev) + if err != nil { + return errors.New("Could not find BPFFS") + } + + if _, err := os.Stat(bpffs); os.IsNotExist(err) { + return errors.Wrapf(err, "Error finding BPFFS directory %s doesn't exist: %v", bpffs, err.Error()) + } + + if err = syscall.Unmount(bpffs, 0); err != nil { + return errors.Wrapf(err, "failed to umount %s: %v", bpffs, err.Error()) + } + + if err := os.Remove(bpffs); err != nil { + return errors.Wrapf(err, "Error Remove BPFFS directory %s: %v", bpffs, err.Error()) + } + + logging.Infof("Deleted BPFFS mount point at %s", bpffs) + + return nil +} diff --git a/internal/bpf/xdp-afxdp-redirect/Makefile b/internal/bpf/xdp-afxdp-redirect/Makefile new file mode 100644 index 00000000..c0327e77 --- /dev/null +++ b/internal/bpf/xdp-afxdp-redirect/Makefile @@ -0,0 +1,33 @@ +# Copyright(c) Red Hat Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LLC ?= llc +CLANG ?= clang + +all: afxdp_redirect + +afxdp_redirect: + $(CLANG) -S \ + -target bpf \ + -D __BPF_TRACING__ \ + -I/usr/include/bpf \ + -Wall \ + -Wno-unused-value \ + -Wno-pointer-sign \ + -Wno-compare-distinct-pointer-types \ + -Werror \ + -O2 -emit-llvm -c -g -o xdp_afxdp_redirect.ll xdp_afxdp_redirect.c + $(LLC) -march=bpf -filetype=obj -o xdp_afxdp_redirect.o xdp_afxdp_redirect.ll + +clean: + rm -f *.o xdp_afxdp_redirect.ll diff --git a/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.c b/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.c new file mode 100644 index 00000000..5c114af8 --- /dev/null +++ b/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.c @@ -0,0 +1,41 @@ +/* + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// clang-format off +#include +#include +#include +// clang-format on + +struct { + __uint(type, BPF_MAP_TYPE_XSKMAP); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 64); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} xsks_map SEC(".maps"); + +SEC("xdp") +int xdp_afxdp_redirect(struct xdp_md *ctx) { + int index = ctx->rx_queue_index; + + /* A set entry here means that the correspnding queue_id + * has an active AF_XDP socket bound to it. */ + if (bpf_map_lookup_elem(&xsks_map, &index)) + return bpf_redirect_map(&xsks_map, index, 0); + + return XDP_PASS; +} + +char _license[] SEC("license") = "Dual BSD"; diff --git a/internal/bpf/xdp-pass/xdp_pass.c b/internal/bpf/xdp-pass/xdp_pass.c index a96d26bd..b1ed2487 100644 --- a/internal/bpf/xdp-pass/xdp_pass.c +++ b/internal/bpf/xdp-pass/xdp_pass.c @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/internal/cni/cni.go b/internal/cni/cni.go index a6d062d3..b5841a2c 100644 --- a/internal/cni/cni.go +++ b/internal/cni/cni.go @@ -18,6 +18,11 @@ package cni import ( "encoding/json" "fmt" + "os" + "regexp" + "runtime" + "strings" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" @@ -26,15 +31,12 @@ import ( validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/intel/afxdp-plugins-for-kubernetes/constants" "github.com/intel/afxdp-plugins-for-kubernetes/internal/bpf" + dpcnisyncer "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerclient" "github.com/intel/afxdp-plugins-for-kubernetes/internal/host" "github.com/intel/afxdp-plugins-for-kubernetes/internal/logformats" "github.com/intel/afxdp-plugins-for-kubernetes/internal/networking" logging "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "os" - "regexp" - "runtime" - "strings" ) var bpfHandler = bpf.NewHandler() @@ -51,6 +53,7 @@ type NetConfig struct { LogFile string `json:"logFile,omitempty"` LogLevel string `json:"logLevel,omitempty"` EthtoolCmds []string `json:"ethtoolCmds,omitempty"` + DPSyncer bool `json:"dpSyncer,omitempty"` } func init() { @@ -362,6 +365,14 @@ func CmdDel(args *skel.CmdArgs) error { } } + if cfg.DPSyncer { + logging.Infof("cmdDel(): Asking Device Plugin to delete any BPF maps for %s", cfg.Device) + err := dpcnisyncer.DeleteNetDev(cfg.Device) + if err != nil { + logging.Errorf("cmdDel(): DeleteNetDev from Syncer Server Failed for %s: %v", cfg.Device, err) + } + } + if cfg.Mode == "cdq" { isSf, err := netHandler.IsCdqSubfunction(cfg.Device) if err != nil { @@ -494,4 +505,4 @@ func extractIP(result *current.Result) (string, error) { err := fmt.Errorf("extractIP(): ip is an empty string") return resultIP, err -} \ No newline at end of file +} diff --git a/internal/deviceplugin/config.go b/internal/deviceplugin/config.go index d43280cb..b2dc49ff 100644 --- a/internal/deviceplugin/config.go +++ b/internal/deviceplugin/config.go @@ -18,11 +18,13 @@ package deviceplugin import ( "encoding/json" + "errors" "io/ioutil" "regexp" "strconv" "github.com/intel/afxdp-plugins-for-kubernetes/constants" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver" "github.com/intel/afxdp-plugins-for-kubernetes/internal/host" "github.com/intel/afxdp-plugins-for-kubernetes/internal/networking" "github.com/intel/afxdp-plugins-for-kubernetes/internal/tools" @@ -32,6 +34,7 @@ import ( var ( network networking.Handler node host.Handler + dpcniserver *dpcnisyncerserver.SyncerServer cfgFile *configFile hostDevices map[string]*networking.Device ) @@ -52,15 +55,17 @@ It contains pool specific details, such as mode and the device list. This object is passed into the PoolManager. */ type PoolConfig struct { - Name string // the name of the pool, used for logging and advertising resource to K8s. Pods will request this resource - Mode string // the mode that this pool operates in - Devices map[string]*networking.Device // a map of devices that the pool will manage - UdsServerDisable bool // a boolean to say if pods in this pool require BPF loading the UDS server - UdsTimeout int // timeout value in seconds for the UDS sockets, user provided or defaults to value from constants package - UdsFuzz bool // a boolean to turn on fuzz testing within the UDS server, has no use outside of development and testing - RequiresUnprivilegedBpf bool // a boolean to say if this pool requires unprivileged BPF - UID int // the id of the pod user, we give this user ACL access to the UDS socket - + Name string // the name of the pool, used for logging and advertising resource to K8s. Pods will request this resource + Mode string // the mode that this pool operates in + Devices map[string]*networking.Device // a map of devices that the pool will manage + UdsServerDisable bool // a boolean to say if pods in this pool require BPF loading the UDS server + BpfMapPinningEnable bool // a boolean to say if pods in this pool require BPF map pinning + UdsTimeout int // timeout value in seconds for the UDS sockets, user provided or defaults to value from constants package + UdsFuzz bool // a boolean to turn on fuzz testing within the UDS server, has no use outside of development and testing + RequiresUnprivilegedBpf bool // a boolean to say if this pool requires unprivileged BPF + UID int // the id of the pod user, we give this user ACL access to the UDS socket + EthtoolCmds []string // list of ethtool filters to apply to the netdev + DPCNIServer *dpcnisyncerserver.SyncerServer // grpc syncer between DP and CNI } /* @@ -90,10 +95,16 @@ func GetPluginConfig(configFile string) (PluginConfig, error) { GetPoolConfigs returns a slice of PoolConfig objects. Each object containing the config and device list for one pool. */ -func GetPoolConfigs(configFile string, net networking.Handler, host host.Handler) ([]PoolConfig, error) { +func GetPoolConfigs(configFile string, net networking.Handler, host host.Handler, server *dpcnisyncerserver.SyncerServer) ([]PoolConfig, error) { var poolConfigs []PoolConfig network = net node = host + dpcniserver = server + + if dpcniserver == nil { + logging.Error("Error dpcniserver not configured") + return poolConfigs, errors.New("error no dpcniserver") + } if cfgFile == nil { if err := readConfigFile(configFile); err != nil { @@ -247,13 +258,14 @@ func GetPoolConfigs(configFile string, net networking.Handler, host host.Handler Mode: pool.Mode, Devices: devices, UdsServerDisable: pool.UdsServerDisable, + BpfMapPinningEnable: pool.BpfMapPinningEnable, UdsTimeout: pool.UdsTimeout, UdsFuzz: pool.UdsFuzz, RequiresUnprivilegedBpf: pool.RequiresUnprivilegedBpf, UID: pool.UID, + DPCNIServer: dpcniserver, }) } - } return poolConfigs, nil diff --git a/internal/deviceplugin/configFile.go b/internal/deviceplugin/configFile.go index 78f324b5..c1e53392 100644 --- a/internal/deviceplugin/configFile.go +++ b/internal/deviceplugin/configFile.go @@ -88,6 +88,7 @@ type configFile_Pool struct { Devices []*configFile_Device `json:"Devices"` Nodes []*configFile_Node `json:"Nodes"` UdsServerDisable bool `json:"UdsServerDisable"` + BpfMapPinningEnable bool `json:"BpfMapPinningEnable"` UdsTimeout int `json:"UdsTimeout"` UdsFuzz bool `json:"UdsFuzz"` RequiresUnprivilegedBpf bool `json:"RequiresUnprivilegedBpf"` diff --git a/internal/deviceplugin/poolManager.go b/internal/deviceplugin/poolManager.go index d7c440fe..15414765 100644 --- a/internal/deviceplugin/poolManager.go +++ b/internal/deviceplugin/poolManager.go @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,6 +26,7 @@ import ( "github.com/intel/afxdp-plugins-for-kubernetes/constants" "github.com/intel/afxdp-plugins-for-kubernetes/internal/bpf" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver" "github.com/intel/afxdp-plugins-for-kubernetes/internal/networking" "github.com/intel/afxdp-plugins-for-kubernetes/internal/tools" "github.com/intel/afxdp-plugins-for-kubernetes/internal/udsserver" @@ -41,36 +42,46 @@ PoolManager represents an manages the pool of devices. Each PoolManager registers with Kubernetes as a different device type. */ type PoolManager struct { - Name string - Mode string - Devices map[string]*networking.Device - UpdateSignal chan bool - DpAPISocket string - DpAPIEndpoint string - UdsServerDisable bool - UdsTimeout int - DevicePrefix string - UdsFuzz bool - UID string - DpAPIServer *grpc.Server - ServerFactory udsserver.ServerFactory - BpfHandler bpf.Handler - NetHandler networking.Handler + Name string + Mode string + Devices map[string]*networking.Device + UpdateSignal chan bool + DpAPISocket string + DpAPIEndpoint string + UdsServerDisable bool + BpfMapPinningEnable bool + UdsTimeout int + DevicePrefix string + UdsFuzz bool + UID string + EthtoolFilters []string + DpAPIServer *grpc.Server + ServerFactory udsserver.ServerFactory + MapManagerFactory bpf.MapManagerFactory + BpfHandler bpf.Handler + NetHandler networking.Handler + DpCniSyncerServer *dpcnisyncerserver.SyncerServer + DpCniSyncerSocket string + SyncerActive bool + Pbm bpf.PoolBpfMapManager } func NewPoolManager(config PoolConfig) PoolManager { return PoolManager{ - Name: config.Name, - Mode: config.Mode, - Devices: config.Devices, - UpdateSignal: make(chan bool), - DpAPISocket: pluginapi.DevicePluginPath + constants.Plugins.DevicePlugin.DevicePrefix + "-" + config.Name + ".sock", - DpAPIEndpoint: constants.Plugins.DevicePlugin.DevicePrefix + "-" + config.Name + ".sock", - UdsServerDisable: config.UdsServerDisable, - UdsTimeout: config.UdsTimeout, - DevicePrefix: constants.Plugins.DevicePlugin.DevicePrefix, - UdsFuzz: config.UdsFuzz, - UID: strconv.Itoa(config.UID), + Name: config.Name, + Mode: config.Mode, + Devices: config.Devices, + UpdateSignal: make(chan bool), + DpAPISocket: pluginapi.DevicePluginPath + constants.Plugins.DevicePlugin.DevicePrefix + "-" + config.Name + ".sock", + DpAPIEndpoint: constants.Plugins.DevicePlugin.DevicePrefix + "-" + config.Name + ".sock", + UdsServerDisable: config.UdsServerDisable, + BpfMapPinningEnable: config.BpfMapPinningEnable, + UdsTimeout: config.UdsTimeout, + DevicePrefix: constants.Plugins.DevicePlugin.DevicePrefix, + UdsFuzz: config.UdsFuzz, + UID: strconv.Itoa(config.UID), + EthtoolFilters: config.EthtoolCmds, + DpCniSyncerServer: config.DPCNIServer, } } @@ -79,6 +90,7 @@ Init is called it initialise the PoolManager. */ func (pm *PoolManager) Init(config PoolConfig) error { pm.ServerFactory = udsserver.NewServerFactory() + pm.MapManagerFactory = bpf.NewMapMangerFactory() pm.BpfHandler = bpf.NewHandler() pm.NetHandler = networking.NewHandler() @@ -92,6 +104,21 @@ func (pm *PoolManager) Init(config PoolConfig) error { } logging.Infof("Pool "+pm.DevicePrefix+"/%s registered with Kubelet", pm.Name) + if pm.BpfMapPinningEnable { + var err error + + logging.Infof("Creating new BPF Map manager %s %s", pm.DevicePrefix+"-maps/", pm.UID) + pm.Pbm.Manager, pm.Pbm.Path, err = pm.MapManagerFactory.CreateMapManager(pm.DevicePrefix+"-maps/", pm.UID) + if err != nil { + logging.Errorf("Error new BPF Map manager: %v", err) + return err + } + + logging.Debug("REGISTER MAP MANAGER WITH THE DP<=>CNI grpc Syncer") + pm.DpCniSyncerServer.RegisterMapManager(pm.Pbm) + pm.DpCniSyncerServer.BpfMapPinEnable = true + } + if len(pm.Devices) > 0 { pm.UpdateSignal <- true } @@ -109,6 +136,10 @@ func (pm *PoolManager) Terminate() error { } logging.Infof(pm.DevicePrefix + "/" + pm.Name + " terminated") + if pm.DpCniSyncerServer != nil { + pm.DpCniSyncerServer.StopGRPCSyncer() + } + return nil } @@ -215,6 +246,32 @@ func (pm *PoolManager) Allocate(ctx context.Context, logging.Infof("BPF program loaded on: %s File descriptor: %s", device.Name(), strconv.Itoa(fd)) udsServer.AddDevice(device.Name(), fd) } + + if pm.BpfMapPinningEnable { + logging.Infof("Loading BPF program on device: %s and pinning the map", device.Name()) + pinPath, err := pm.Pbm.Manager.CreateBPFFS(device.Name(), pm.Pbm.Path) + if err != nil { + logging.Errorf("Error Creating the BPFFS: %v", err) + return &response, err + } + + err = pm.BpfHandler.LoadBpfPinXskMap(device.Name(), pinPath) + if err != nil { + logging.Errorf("Error loading BPF Program on interface %s and pinning the map: %v", device.Name(), err) + return &response, err + } + + pm.Pbm.Manager.AddMap(device.Name(), pinPath) + + //FULL PATH WILL INCLUDE THE XSKMAP... + fullPath := pinPath + constants.Bpf.Xsk_map + logging.Debugf("mapping %s to %s", fullPath, constants.Bpf.BpfMapPodPath) + cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{ + HostPath: fullPath, + ContainerPath: constants.Bpf.BpfMapPodPath, + ReadOnly: false, + }) + } } envs[constants.Devices.EnvVarList] = strings.Join(crqt.DevicesIDs, " ") diff --git a/internal/deviceplugin/poolManager_test.go b/internal/deviceplugin/poolManager_test.go index 827b3c04..3e6be008 100644 --- a/internal/deviceplugin/poolManager_test.go +++ b/internal/deviceplugin/poolManager_test.go @@ -46,6 +46,7 @@ func TestAllocate(t *testing.T) { "dev_9": networking.CreateTestDevice("dev_9", "primary", "ice", "0000:81:00.9", "68:05:ca:2d:e9:09", netHandler), }, UdsServerDisable: false, + BpfMapPinningEnable: false, UdsTimeout: 0, UdsFuzz: false, RequiresUnprivilegedBpf: false, diff --git a/internal/dpcnisyncer/dp_cni_syncer.pb.go b/internal/dpcnisyncer/dp_cni_syncer.pb.go new file mode 100644 index 00000000..d5cdd932 --- /dev/null +++ b/internal/dpcnisyncer/dp_cni_syncer.pb.go @@ -0,0 +1,140 @@ +/* + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//lint:file-ignore SA1019 Ignore all deprecated code, it's generated +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: dp_cni_syncer.proto + +package dpcnisyncer + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type DeleteNetDevReq struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteNetDevReq) Reset() { *m = DeleteNetDevReq{} } +func (m *DeleteNetDevReq) String() string { return proto.CompactTextString(m) } +func (*DeleteNetDevReq) ProtoMessage() {} +func (*DeleteNetDevReq) Descriptor() ([]byte, []int) { + return fileDescriptor_f9ab154255673e38, []int{0} +} + +func (m *DeleteNetDevReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteNetDevReq.Unmarshal(m, b) +} +func (m *DeleteNetDevReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteNetDevReq.Marshal(b, m, deterministic) +} +func (m *DeleteNetDevReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteNetDevReq.Merge(m, src) +} +func (m *DeleteNetDevReq) XXX_Size() int { + return xxx_messageInfo_DeleteNetDevReq.Size(m) +} +func (m *DeleteNetDevReq) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteNetDevReq.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteNetDevReq proto.InternalMessageInfo + +func (m *DeleteNetDevReq) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type DeleteNetDevResp struct { + Ret int32 `protobuf:"varint,1,opt,name=ret,proto3" json:"ret,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteNetDevResp) Reset() { *m = DeleteNetDevResp{} } +func (m *DeleteNetDevResp) String() string { return proto.CompactTextString(m) } +func (*DeleteNetDevResp) ProtoMessage() {} +func (*DeleteNetDevResp) Descriptor() ([]byte, []int) { + return fileDescriptor_f9ab154255673e38, []int{1} +} + +func (m *DeleteNetDevResp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteNetDevResp.Unmarshal(m, b) +} +func (m *DeleteNetDevResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteNetDevResp.Marshal(b, m, deterministic) +} +func (m *DeleteNetDevResp) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteNetDevResp.Merge(m, src) +} +func (m *DeleteNetDevResp) XXX_Size() int { + return xxx_messageInfo_DeleteNetDevResp.Size(m) +} +func (m *DeleteNetDevResp) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteNetDevResp.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteNetDevResp proto.InternalMessageInfo + +func (m *DeleteNetDevResp) GetRet() int32 { + if m != nil { + return m.Ret + } + return 0 +} + +func init() { + proto.RegisterType((*DeleteNetDevReq)(nil), "dpcnisyncer.DeleteNetDevReq") + proto.RegisterType((*DeleteNetDevResp)(nil), "dpcnisyncer.DeleteNetDevResp") +} + +func init() { + proto.RegisterFile("dp_cni_syncer.proto", fileDescriptor_f9ab154255673e38) +} + +var fileDescriptor_f9ab154255673e38 = []byte{ + // 202 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4e, 0x29, 0x88, 0x4f, + 0xce, 0xcb, 0x8c, 0x2f, 0xae, 0xcc, 0x4b, 0x4e, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, + 0xe2, 0x4e, 0x29, 0x48, 0xce, 0xcb, 0x84, 0x08, 0x29, 0xa9, 0x72, 0xf1, 0xbb, 0xa4, 0xe6, 0xa4, + 0x96, 0xa4, 0xfa, 0xa5, 0x96, 0xb8, 0xa4, 0x96, 0x05, 0xa5, 0x16, 0x0a, 0x09, 0x71, 0xb1, 0xe4, + 0x25, 0xe6, 0xa6, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x4a, 0x2a, 0x5c, 0x02, + 0xa8, 0xca, 0x8a, 0x0b, 0x84, 0x04, 0xb8, 0x98, 0x8b, 0x52, 0x4b, 0xc0, 0xca, 0x58, 0x83, 0x40, + 0x4c, 0xa3, 0x20, 0x2e, 0x36, 0x88, 0xbc, 0x90, 0x07, 0x17, 0xa7, 0x4b, 0x6a, 0x0e, 0x94, 0x23, + 0xa3, 0x87, 0x64, 0xa3, 0x1e, 0x9a, 0x75, 0x52, 0xb2, 0x78, 0x64, 0x8b, 0x0b, 0x9c, 0x5c, 0xa2, + 0x9c, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0x33, 0xf3, 0x4a, 0x52, + 0x73, 0xf4, 0x13, 0xd3, 0x2a, 0x52, 0x0a, 0x74, 0x0b, 0x72, 0x4a, 0xd3, 0x33, 0xf3, 0x8a, 0x75, + 0xd3, 0xf2, 0x8b, 0x74, 0xb3, 0x4b, 0x93, 0x52, 0x8b, 0xf2, 0x52, 0x4b, 0x52, 0x8b, 0xc1, 0x4a, + 0x8a, 0xf2, 0x12, 0x73, 0xf4, 0x91, 0x8c, 0x4d, 0x62, 0x03, 0x7b, 0xdd, 0x18, 0x10, 0x00, 0x00, + 0xff, 0xff, 0x74, 0x57, 0xca, 0xf4, 0x11, 0x01, 0x00, 0x00, +} diff --git a/internal/dpcnisyncer/dp_cni_syncer.proto b/internal/dpcnisyncer/dp_cni_syncer.proto new file mode 100644 index 00000000..1de4cda7 --- /dev/null +++ b/internal/dpcnisyncer/dp_cni_syncer.proto @@ -0,0 +1,32 @@ +/* + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option go_package = "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncer"; + +package dpcnisyncer_proto; + +service NetDev { + rpc DelNetDev(DeleteNetDevReq) returns (DeleteNetDevResp); +} + +message DeleteNetDevReq { + string name = 1; +} + +message DeleteNetDevResp { + int32 ret = 1; +} diff --git a/internal/dpcnisyncer/dp_cni_syncer_grpc.pb.go b/internal/dpcnisyncer/dp_cni_syncer_grpc.pb.go new file mode 100644 index 00000000..fa8e49a3 --- /dev/null +++ b/internal/dpcnisyncer/dp_cni_syncer_grpc.pb.go @@ -0,0 +1,124 @@ +/* + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 +// source: dp_cni_syncer.proto + +package dpcnisyncer + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + NetDev_DelNetDev_FullMethodName = "/dpcnisyncer.NetDev/DelNetDev" +) + +// NetDevClient is the client API for NetDev service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type NetDevClient interface { + DelNetDev(ctx context.Context, in *DeleteNetDevReq, opts ...grpc.CallOption) (*DeleteNetDevResp, error) +} + +type netDevClient struct { + cc grpc.ClientConnInterface +} + +func NewNetDevClient(cc grpc.ClientConnInterface) NetDevClient { + return &netDevClient{cc} +} + +func (c *netDevClient) DelNetDev(ctx context.Context, in *DeleteNetDevReq, opts ...grpc.CallOption) (*DeleteNetDevResp, error) { + out := new(DeleteNetDevResp) + err := c.cc.Invoke(ctx, NetDev_DelNetDev_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// NetDevServer is the server API for NetDev service. +// All implementations must embed UnimplementedNetDevServer +// for forward compatibility +type NetDevServer interface { + DelNetDev(context.Context, *DeleteNetDevReq) (*DeleteNetDevResp, error) + mustEmbedUnimplementedNetDevServer() +} + +// UnimplementedNetDevServer must be embedded to have forward compatible implementations. +type UnimplementedNetDevServer struct { +} + +func (UnimplementedNetDevServer) DelNetDev(context.Context, *DeleteNetDevReq) (*DeleteNetDevResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method DelNetDev not implemented") +} +func (UnimplementedNetDevServer) mustEmbedUnimplementedNetDevServer() {} + +// UnsafeNetDevServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NetDevServer will +// result in compilation errors. +type UnsafeNetDevServer interface { + mustEmbedUnimplementedNetDevServer() +} + +func RegisterNetDevServer(s grpc.ServiceRegistrar, srv NetDevServer) { + s.RegisterService(&NetDev_ServiceDesc, srv) +} + +func _NetDev_DelNetDev_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteNetDevReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NetDevServer).DelNetDev(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: NetDev_DelNetDev_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NetDevServer).DelNetDev(ctx, req.(*DeleteNetDevReq)) + } + return interceptor(ctx, in, info, handler) +} + +// NetDev_ServiceDesc is the grpc.ServiceDesc for NetDev service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NetDev_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "dpcnisyncer.NetDev", + HandlerType: (*NetDevServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "DelNetDev", + Handler: _NetDev_DelNetDev_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "dp_cni_syncer.proto", +} diff --git a/internal/dpcnisyncerclient/client.go b/internal/dpcnisyncerclient/client.go new file mode 100644 index 00000000..1e666e68 --- /dev/null +++ b/internal/dpcnisyncerclient/client.go @@ -0,0 +1,59 @@ +/* + * Copyright(c) 2023 Intel Corporation. + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dpcnisyncerclient + +import ( + "context" + "net" + + "github.com/intel/afxdp-plugins-for-kubernetes/constants" + pb "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncer" + logging "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +const ( + _proto = "unix" +) + +var ( + sock = pluginapi.DevicePluginPath + constants.Plugins.DevicePlugin.DevicePrefix + "-" + "syncer.sock" +) + +func DeleteNetDev(name string) error { + ctx := context.Background() + conn, err := grpc.DialContext(ctx, sock, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, _proto, addr) + })) + if err != nil { + logging.Errorf("error connecting to Server") + return err + } + defer conn.Close() + + c := pb.NewNetDevClient(conn) + r, err := c.DelNetDev(ctx, &pb.DeleteNetDevReq{Name: name}) + if err != nil || r.Ret == -1 { + logging.Errorf("error deleting netdev resources for netdev %s", name) + return err + } + logging.Infof("Server response:%v", r) + + return nil +} diff --git a/internal/dpcnisyncerserver/server.go b/internal/dpcnisyncerserver/server.go new file mode 100644 index 00000000..9116d15b --- /dev/null +++ b/internal/dpcnisyncerserver/server.go @@ -0,0 +1,153 @@ +/* + * Copyright(c) 2023 Intel Corporation. + * Copyright(c) Red Hat Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dpcnisyncerserver + +import ( + "context" + "net" + "os" + "time" + + "github.com/intel/afxdp-plugins-for-kubernetes/constants" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/bpf" + pb "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncer" + "github.com/pkg/errors" + logging "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +const ( + protocol = "unix" +) + +var ( + sockAddr = pluginapi.DevicePluginPath + constants.Plugins.DevicePlugin.DevicePrefix + "-" + "syncer.sock" +) + +type SyncerServer struct { + pb.UnimplementedNetDevServer + mapManagers []bpf.PoolBpfMapManager + grpcServer *grpc.Server + BpfMapPinEnable bool +} + +func (s *SyncerServer) RegisterMapManager(b bpf.PoolBpfMapManager) { + + if s.mapManagers != nil { + for _, v := range s.mapManagers { + if v.Manager.GetName() == b.Manager.GetName() { + logging.Infof("%s is already registered", b.Manager.GetName()) + return + } + } + } + + s.mapManagers = append(s.mapManagers, b) +} + +func (s *SyncerServer) DelNetDev(ctx context.Context, in *pb.DeleteNetDevReq) (*pb.DeleteNetDevResp, error) { + + if s.BpfMapPinEnable { + netDevName := in.GetName() + + logging.Infof("Looking up Map Manager for %s", netDevName) + found := false + var pm bpf.PoolBpfMapManager + for _, mm := range s.mapManagers { + _, err := mm.Manager.GetBPFFS(netDevName) + if err == nil { + found = true + pm = mm + break + } + } + + if !found { + logging.Errorf("Could NOT find the map manager for device %s", netDevName) + return &pb.DeleteNetDevResp{Ret: -1}, errors.New("Could NOT find the map manager for device") + } + + logging.Infof("Map Manager found, deleting BPFFS for %s", netDevName) + err := pm.Manager.DeleteBPFFS(netDevName) + if err != nil { + logging.Errorf("Could NOT delete BPFFS for %s", netDevName) + return &pb.DeleteNetDevResp{Ret: -1}, errors.Wrapf(err, "Could NOT delete BPFFS for %s: %v", netDevName, err.Error()) + } + + logging.Infof("Network interface %s deleted", netDevName) + return &pb.DeleteNetDevResp{Ret: 0}, nil + } + + return &pb.DeleteNetDevResp{Ret: -1}, errors.New("BPF Map pinning is not enabled") +} + +func (s *SyncerServer) StopGRPCSyncer() { + if s.grpcServer != nil { + s.grpcServer.Stop() + s.grpcServer = nil + } + s.cleanup() +} + +func NewSyncerServer() (*SyncerServer, error) { + if _, err := os.Stat(sockAddr); !os.IsNotExist(err) { + if err := os.RemoveAll(sockAddr); err != nil { + logging.Errorf("sockAddr %s does not exist", sockAddr) + return nil, err + } + } + + server := &SyncerServer{ + grpcServer: grpc.NewServer(), + BpfMapPinEnable: false, + } + + lis, err := net.Listen(protocol, sockAddr) + if err != nil { + logging.Errorf("Could not listen to %s", sockAddr) + return nil, err + } + + pb.RegisterNetDevServer(server.grpcServer, server) + go func() { + if err := server.grpcServer.Serve(lis); err != nil { + logging.Errorf("Could not RegisterNetDevServer: %v", err) + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := grpc.DialContext(ctx, sockAddr, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", addr) + }), + ) + if err != nil { + logging.Errorf("Unable to establish test connection with gRPC server: %v", err) + return nil, err + } + conn.Close() + logging.Debugf("NewSyncerServer up and Running") + return server, nil +} + +func (s *SyncerServer) cleanup() error { + if err := os.Remove(sockAddr); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} diff --git a/test/fuzz/deviceplugin/config/config.go b/test/fuzz/deviceplugin/config/config.go index 4caf203f..7a2eea23 100644 --- a/test/fuzz/deviceplugin/config/config.go +++ b/test/fuzz/deviceplugin/config/config.go @@ -16,11 +16,13 @@ package deviceplugin import ( + "io/ioutil" + "os" + dp "github.com/intel/afxdp-plugins-for-kubernetes/internal/deviceplugin" + "github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver" "github.com/intel/afxdp-plugins-for-kubernetes/internal/host" "github.com/intel/afxdp-plugins-for-kubernetes/internal/networking" - "io/ioutil" - "os" ) const ( @@ -61,7 +63,13 @@ func Fuzz(data []byte) int { panic(1) } - _, err = dp.GetPoolConfigs(tmpfile.Name(), networking.NewHandler(), host.NewHandler()) + //START THE SYNCER SERVER TODO CHECK BPF MAP + dpCniSyncerServer, err := dpcnisyncerserver.NewSyncerServer() + if err != nil { + panic(1) + } + + _, err = dp.GetPoolConfigs(tmpfile.Name(), networking.NewHandler(), host.NewHandler(), dpCniSyncerServer) if err != nil { return 0 }