bpf-map-pinning: fixes a few issues

* Update the documentation to reflect configuration
changes needed.
* Update the program to pin the xsk map to just use
xdp-loader. The previous c implementation had issues
and the version of libxdp in the container images is
too old for libxdp pinning support. Reverting to bpf
(for pinning support) would break the libxdp based
program unloading.
* Don't return an error from the bpf clean up function
when there's no program on the interface.

Signed-off-by: Maryam Tahhan <[email protected]>

README.md

@@ -411,6 +411,8 @@ UdsServerDisable is a Boolean configuration. If set to true, devices in this poo
 
 BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration.
 
+> **_NOTE:_**  When using this setting it's important to adjust the securityContext for the DP Daemonset to `privileged: true` to allow for bidirectional propagation of the volume where the BPF maps are pinned. this will no longer be needed when the DP is integrated with bpfman.
+
 #### UdsTimeout
 
 UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds.

cmd/deviceplugin/main.go

@@ -143,7 +143,6 @@ func main() {
 			logging.Errorf("Termination error: %v", err)
 		}
 	}
-
 }
 
 func configureLogging(cfg deviceplugin.PluginConfig) error {

constants/constants.go

@@ -84,6 +84,7 @@ var (
 	udsPodSock    = "/afxdp.sock"
 
 	/* BPF*/
+	pinMapBaseDir = "/var/run/afxdp_dp/"
 	bpfMapPodPath = "/tmp/afxdp_dp/"
 	xsk_map       = "/xsks_map"
 
@@ -222,6 +223,7 @@ type uds struct {
 }
 
 type bpf struct {
+	PinMapBaseDir string
 	BpfMapPodPath string
 	Xsk_map       string
 }
@@ -349,6 +351,7 @@ func init() {
 	}
 
 	Bpf = bpf{
+		PinMapBaseDir: pinMapBaseDir,
 		BpfMapPodPath: bpfMapPodPath,
 		Xsk_map:       xsk_map,
 	}

deployments/daemonset.yml

@@ -62,6 +62,7 @@ spec:
           image: intel/afxdp-plugins-for-kubernetes:latest
           imagePullPolicy: IfNotPresent
           securityContext:
+            #privileged: true # NEEDED WHEN USING BPF MAP PINNING FOR BIDIR PROPAGATION
             capabilities:
               drop:
                 - all
@@ -78,6 +79,9 @@ spec:
           volumeMounts:
             - name: unixsock
               mountPath: /tmp/afxdp_dp/
+            - name: bpfmappinning
+              mountPath: /var/run/afxdp_dp/
+              #mountPropagation: Bidirectional
             - name: devicesock
               mountPath: /var/lib/kubelet/device-plugins/
             - name: resources
@@ -92,6 +96,9 @@ spec:
         - name: unixsock
           hostPath:
             path: /tmp/afxdp_dp/
+        - name: bpfmappinning
+          hostPath:
+            path: /var/run/afxdp_dp/
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/device-plugins/

images/amd64.dockerfile

@@ -28,13 +28,14 @@ WORKDIR /usr/src/afxdp_k8s_plugins
 RUN apk add --no-cache build-base~=0.5-r3 \
 && apk add --no-cache libbsd-dev~=0.11.7 \
 && apk add --no-cache libxdp-dev~=1.2.10-r0 \
+&& apk add --no-cache libbpf-dev~=1.0.1-r0 \
 && apk add --no-cache llvm15~=15.0.7-r0 \
 && apk add --no-cache clang15~=15.0.7-r0 \
 && make builddp
 
 FROM amd64/alpine:3.18@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70
 RUN apk --no-cache -U add iproute2-rdma~=6.3.0-r0 acl~=2.3 \
-      && apk add --no-cache libxdp~=1.2.10-r0
+      && apk add --no-cache xdp-tools~=1.2.10-r0
 COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh

internal/bpf/bpfWrapper.c

@@ -158,13 +158,13 @@ int Clean_bpf(char *ifname) {
 
 	mp = xdp_multiprog__get_from_ifindex(if_index);
 	if (!mp) {
-		Log_Error("%s: unable to receive correct multi_prog reference : %s", __FUNCTION__,
-			  mp);
-		return -1;
+		Log_Info("%s: No programs loaded on : %s", __FUNCTION__,
+			  ifname);
+		return 0;
 	}
 
 	err = xdp_multiprog__detach(mp);
-	if (err) {
+	if (err && err != -EINVAL) { // -EINVAL == No program attached
 		Log_Error("%s: Removal of xdp program failed, returned: "
 			  "returned: %d",
 			  __FUNCTION__, err);
@@ -180,7 +180,6 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 	char *filename = "/afxdp/xdp_pass.o";
 	struct bpf_object *obj;
 	struct xdp_program *prog;
-	__u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_DRV_MODE;
 
 	Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname);
 
@@ -212,7 +211,7 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 	}
 
 	/* Attach the program to the interface at the xdp hook */
-	err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0);
+	err = xdp_program__attach(prog, ifindex, XDP_MODE_NATIVE, 0);
 	if (err) {
 		libxdp_strerror(err, "Couldn't attach the xdp pass program",
 				sizeof("Couldn't attach the xdp pass program"));
@@ -225,51 +224,79 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 	return 0;
 }
 
-int Load_bpf_pin_xsk_map(char *ifname, char *pin_path) {
-	struct bpf_object *obj;
-	struct xdp_program *prog;
-	struct bpf_link *link;
-	int ifindex, map_fd = -1;
-	int err;
-	const char *prog_name = "xdp_afxdp_redirect";
-	char *filename = "/afxdp/xdp_afxdp_redirect.o";
-	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, bpf_opts, .pin_root_path = pin_path);
-
-	ifindex = if_nametoindex(ifname);
-	if (!ifindex) {
-		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
-		return -1;
-	}
-	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
-
-	if (access(filename, O_RDONLY) < 0) {
-		Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename,
-			  strerror(errno));
-		return err;
-	}
-
-	Log_Info("%s: starting setup of xdp-redirect program on "
-		 "interface %s (%d)",
-		 __FUNCTION__, ifname, ifindex);
-
-	/* Load the BPF program */
-	prog = xdp_program__open_file(filename, NULL, NULL);
-	err = libxdp_get_error(prog);
-	if (err) {
-		libxdp_strerror(err, "Couldn’t load XDP program",
-				sizeof("Couldn’t load XDP program"));
-		Log_Error("%s: Couldn’t load XDP program\n", __FUNCTION__, filename);
-		return err;
-	}
-
-	/* Attach the program to the interface at the xdp hook */
-	err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0);
-	if (err) {
-		libxdp_strerror(err, "Couldn't attach the xdp pass program",
-				sizeof("Couldn't attach the xdp pass program"));
-		Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname);
-		return err;
-	}
-
-	return 0;
-}
+// int Load_bpf_pin_xsk_map(char *ifname, char *pin_path) {
+// 	struct bpf_object *obj;
+// 	struct bpf_program *prog;
+// 	struct bpf_link *link;
+// 	int ifindex, map_fd = -1;
+// 	int err;
+// 	const char *prog_name = "xdp_afxdp_redirect";
+// 	char *filename = "/afxdp/xdp_afxdp_redirect.o";
+// 	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, bpf_opts, .pin_root_path = pin_path);
+
+// 	ifindex = if_nametoindex(ifname);
+// 	if (!ifindex) {
+// 		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
+// 		return -1;
+// 	}
+// 	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+
+// 	if (access(filename, O_RDONLY) < 0) {
+// 		Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename,
+// 			  strerror(errno));
+// 		return err;
+// 	}
+
+// 	obj = bpf_object__open_file(filename, &bpf_opts);
+// 	err = libbpf_get_error(obj);
+// 	if (err) {
+// 		Log_Error("%s: Couldn't open file(%s)\n", __FUNCTION__, filename);
+// 		return err;
+// 	}
+
+// 	prog = bpf_object__find_program_by_name(obj, prog_name);
+// 	if (!prog) {
+// 		Log_Error("%s: Couldn't find xdp program in bpf object!\n", __FUNCTION__);
+// 		err = -ENOENT;
+// 		return err;
+// 	}
+// 	bpf_program__set_type(prog, BPF_PROG_TYPE_XDP);
+
+// 	err = bpf_object__load(obj);
+// 	if (err) {
+// 		Log_Error("%s: Couldn't load BPF-OBJ file(%s) %s\n", __FUNCTION__, filename,
+// 			  strerror(errno));
+// 		return err;
+// 	}
+
+// 	Log_Info("%s: bpf: Attach prog to ifindex %d\n", __FUNCTION__, ifindex);
+// 	link = bpf_program__attach_xdp(prog, ifindex);
+// 	if (!link) {
+// 		Log_Error("%s:ERROR: failed to attach program to %s\n", __FUNCTION__, ifname);
+// 		return err;
+// 	}
+
+// 	if (bpf_map__is_pinned(bpf_object__find_map_by_name(obj, "xsks_map"))) {
+// 		Log_Info("%s: xsk map pinned to %s\n", __FUNCTION__,
+// 			 bpf_map__get_pin_path(bpf_object__find_map_by_name(obj, "xsks_map")));
+// 	} else {
+// 		Log_Error("%s: xsk map is not pinned \n", __FUNCTION__);
+// 		return -1;
+// 	}
+
+// 	return 0;
+// }
+
+// int Unload_bpf_prog(char *ifname) {
+//     int ifindex, prog_id = 0;
+
+//     ifindex = if_nametoindex(ifname);
+// 	if (!ifindex) {
+// 		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
+// 		return -1;
+// 	}
+// 	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+
+//     if (bpf_xdp_query_id(xi->if_index, 0, &prog_id))
+
+// }

internal/bpf/bpfWrapper.go

@@ -26,6 +26,8 @@ import "C"
 
 import (
 	"errors"
+	"fmt"
+	"os/exec"
 
 	logging "github.com/sirupsen/logrus"
 )
@@ -85,10 +87,24 @@ func (r *handler) LoadAttachBpfXdpPass(ifname string) error {
 LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_send_xsk_map
 */
 func (r *handler) LoadBpfPinXskMap(ifname, pin_path string) error {
-	err := int(C.Load_bpf_pin_xsk_map(C.CString(ifname), C.CString(pin_path)))
 
-	if err < 0 {
-		return errors.New("error loading BPF program onto interface")
+	bpfProg := "/afxdp/xdp_afxdp_redirect.o"
+
+	loaderCmd := "xdp-loader load " + ifname + " " + bpfProg + " -p " + pin_path
+
+	if err := r.Cleanbpf(ifname); err != nil {
+		err = fmt.Errorf("cmdDel(): error removing BPF program from device: %w", err)
+		logging.Errorf(err.Error())
+
+		return err
+	}
+
+	logging.Infof("Loading XDP program using: %s", loaderCmd)
+
+	cmd := exec.Command("xdp-loader", "load", ifname, bpfProg, "-p", pin_path)
+	if err := cmd.Run(); err != nil {
+		logging.Errorf("error loading and pinning BPF program onto interface %v", err)
+		return errors.New("error loading and pinning BPF program onto interface")
 	}
 
 	return nil
@@ -111,6 +127,7 @@ func (r *handler) ConfigureBusyPoll(fd int, busyTimeout int, busyBudget int) err
 Cleanbpf is the GoLang wrapper for the C function Clean_bpf
 */
 func (r *handler) Cleanbpf(ifname string) error {
+
 	ret := C.Clean_bpf(C.CString(ifname))
 
 	if ret != 0 {

internal/bpf/bpfWrapper.h

@@ -19,7 +19,7 @@
 
 int Load_bpf_send_xsk_map(char *ifname);
 int Load_attach_bpf_xdp_pass(char *ifname);
-int Load_bpf_pin_xsk_map(char *ifname, char *pin_path);
+// int Load_bpf_pin_xsk_map(char *ifname, char *pin_path);
 int Configure_busy_poll(int fd, int busy_timeout, int busy_budget);
 int Clean_bpf(char *ifname);