Updated the libbpf version and made changes to the relevant files (#74)

* Replaced Libbpf with Libxdp * Renamed HasLibbpf function to HasLibxdp and modified functionality to reflect * CNI statically built so it no longer depends on host libraries * CI updates to accommodate all the above --------- Signed-off-by: phansGithub <[email protected]>
intel · Oct 10, 2023 · 7a50f12 · 7a50f12
1 parent 93d43de
commit 7a50f12
Show file tree

Hide file tree

Showing 9 changed files with 128 additions and 76 deletions.
diff --git a/.github/workflows/public-ci.yml b/.github/workflows/public-ci.yml
@@ -38,8 +38,23 @@ jobs:
       with:
         go-version: 1.19
 
-    - name: Install dependencies
-      run:  sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib
+    - name: Install libbpf and libxdp
+      run: | 
+          sudo apt update 
+          sudo apt install -y wget build-essential golang
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb
+          sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb
+          sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y clang
+          sudo apt install -y llvm
+          sudo apt install -y gcc-multilib
+
+ 
 
     - name: Run build
       run: make build
@@ -54,8 +69,18 @@ jobs:
       with:
         go-version: 1.19
 
-    - name: Install dependencies
-      run:  sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib
+    - name: Install libbpf and libxdp
+      run: | 
+          sudo apt update 
+          sudo apt install -y wget build-essential golang
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb
+          sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb
+          sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb
 
     - name: Install staticcheck
       run: go install honnef.co/go/tools/cmd/staticcheck@latest
@@ -73,8 +98,21 @@ jobs:
       with:
         go-version: 1.19
 
-    - name: Install dependencies
-      run:  sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib
+    - name: Install libbpf and libxdp
+      run: | 
+          sudo apt update 
+          sudo apt install -y wget build-essential golang
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb
+          sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb
+          sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb
+          sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb
+          sudo apt install -y clang
+          sudo apt install -y llvm
+          sudo apt install -y gcc-multilib
 
     - name: unit-tests
       run: make test
diff --git a/Makefile b/Makefile
@@ -40,7 +40,7 @@ format: clangformat
 buildc:
 	@echo "******     Build BPF     ******"
 	@echo
-	gcc ./internal/bpf/bpfWrapper.c -lbpf -c -o ./internal/bpf/bpfWrapper.o
+	gcc ./internal/bpf/bpfWrapper.c -lxdp -c -o ./internal/bpf/bpfWrapper.o
 	ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o  &> /dev/null
 	@echo "******     Build xdp_pass     ******"
 	make -C ./internal/bpf/xdp-pass/
@@ -58,7 +58,7 @@ builddp: buildc
 buildcni: buildc
 	@echo "******     Build CNI     ******"
 	@echo
-	go build -o ./bin/afxdp ./cmd/cni
+	go build -ldflags="-extldflags=-static" -tags netgo -o ./bin/afxdp ./cmd/cni
 	@echo
 	@echo
 
@@ -68,24 +68,24 @@ build: builddp buildcni
 docker: ## Build docker image
 	@echo "******  Docker Image    ******"
 	@echo
-	docker build -t afxdp-device-plugin -f images/amd64.dockerfile .
+	docker build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
 	@echo
 	@echo
 
 podman: ## Build podman image
 	@echo "******  Podman Image    ******"
 	@echo
-	podman build -t afxdp-device-plugin -f images/amd64.dockerfile .
+	podman build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
 	@echo
 	@echo
 
 image:
-	if $(MAKE) podman; then \
-	 echo "Podman build succeeded"; \
-	else \
-	 echo "Podman build failed, trying docker.."; \
+	# if $(MAKE) podman; then \
+	#  echo "Podman build succeeded"; \
+	# else \
+	#  echo "Podman build failed, trying docker.."; \
 	 $(MAKE) docker; \
-	fi
+	# fi
 
 undeploy: ## Undeploy the Deamonset
 	@echo "******  Stop Daemonset   ******"

diff --git a/cmd/deviceplugin/main.go b/cmd/deviceplugin/main.go
@@ -185,21 +185,23 @@ func configureLogging(cfg deviceplugin.PluginConfig) error {
 // On each Kind node
 // Create a bridge afxdp-kind-br
 // Create 4 vethpairs starting at veth6
-//  +===============+
-//  | afxdp-kind-br |
-//  |     +---------|         +---------+
-//  |     |  veth7  | <=====> |  veth6  |
-//  |     +---------|         +---------+
-//  |     +---------|         +---------+
-//  |     |  veth9  | <=====> |  veth8  |
-//  |     +---------|         +---------+
-//  |     +---------|         +---------+
-//  |     |  veth11 | <=====> |  veth10 |
-//  |     +---------|         +---------+
-//  |     +---------|         +---------+
-//  |     |  veth13 | <=====> |  veth12 |
-//  |     +---------|         +---------+
-//  +===============+
+//
+//	+===============+
+//	| afxdp-kind-br |
+//	|     +---------|         +---------+
+//	|     |  veth7  | <=====> |  veth6  |
+//	|     +---------|         +---------+
+//	|     +---------|         +---------+
+//	|     |  veth9  | <=====> |  veth8  |
+//	|     +---------|         +---------+
+//	|     +---------|         +---------+
+//	|     |  veth11 | <=====> |  veth10 |
+//	|     +---------|         +---------+
+//	|     +---------|         +---------+
+//	|     |  veth13 | <=====> |  veth12 |
+//	|     +---------|         +---------+
+//	+===============+
+//
 // The "even" veth of the pair will be added to the device plugin resource pool.
 // and plumbed to the Pod.
 func configureKindSecondaryNetwork() error {
@@ -249,7 +251,7 @@ func checkHost(host host.Handler) (bool, error) {
 
 	// libbpf
 	logging.Debugf("Checking host for Libbpf")
-	bpfInstalled, libs, err := host.HasLibbpf()
+	bpfInstalled, libs, err := host.HasLibxdp()
 	if err != nil {
 		err := fmt.Errorf("libbpf not found on host")
 		return false, err

diff --git a/images/amd64.dockerfile b/images/amd64.dockerfile
@@ -12,24 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20@sha256:52921e63cc544c79c111db1d8461d8ab9070992d9c636e1573176642690c14b5 as cnibuilder
+FROM golang:1.20@sha256:efe38cb419e2b2012f66d1782d2efe2fd8884c71d9f342581e1697ba9047b5f8 as cnibuilder
 COPY . /usr/src/afxdp_k8s_plugins
 WORKDIR /usr/src/afxdp_k8s_plugins
-RUN apt-get update && apt-get -y install --no-install-recommends libbpf-dev=1:0.3-2 \
-       && apt-get -y install --no-install-recommends clang=1:11.0-51+nmu5 llvm=1:11.0-51+nmu5 gcc-multilib=4:10.2.1-1 \
-       && make buildcni
+RUN apt-get update \
+&& apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6  \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 \
+&& make buildcni
 
-FROM golang:1.20-alpine@sha256:87d0a3309b34e2ca732efd69fb899d3c420d3382370fd6e7e6d2cb5c930f27f9 as dpbuilder
+FROM golang:1.20-alpine@sha256:ebceb16dc094769b6e2a393d51e0417c19084ba20eb8967fb3f7675c32b45774 as dpbuilder
 COPY . /usr/src/afxdp_k8s_plugins
 WORKDIR /usr/src/afxdp_k8s_plugins
-RUN apk add --no-cache build-base~=0.5 libbsd-dev~=0.11 \
-      && apk add --no-cache libbpf-dev~=0.5 --repository=https://dl-cdn.alpinelinux.org/alpine/v3.15/community \
-      && apk add --no-cache llvm~=15.0.7-r0 clang~=15.0.7-r0 \
-	  && make builddp
+RUN apk add --no-cache build-base~=0.5-r3 \
+&& apk add --no-cache libbsd-dev~=0.11.7 \
+&& apk add --no-cache libxdp-dev~=1.2.10-r0 \
+&& apk add --no-cache llvm15~=15.0.7-r0 \
+&& apk add --no-cache clang15~=15.0.7-r0 \
+&& make builddp
 
-FROM amd64/alpine:3.17@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501
-RUN apk --no-cache -U add iproute2-rdma~=6.0 acl~=2.3 \
-      && apk --no-cache -U add libbpf~=0.5 --repository=http://dl-cdn.alpinelinux.org/alpine/v3.15/community
+FROM amd64/alpine:3.18@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70
+RUN apk --no-cache -U add iproute2-rdma~=6.3.0-r0 acl~=2.3 \
+      && apk add --no-cache libxdp~=1.2.10-r0
 COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh

diff --git a/internal/bpf/bpfWrapper.c b/internal/bpf/bpfWrapper.c
@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-#include <bpf/xsk.h>	   // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd
 #include <linux/if_link.h> // for XDP_FLAGS_DRV_MODE
 #include <net/if.h>	   // for if_nametoindex
+#include <xdp/libxdp.h>
+#include <xdp/xsk.h> // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd
 
 #include "bpfWrapper.h"
 #include "log.h"
@@ -40,12 +40,12 @@ int Load_bpf_send_xsk_map(char *ifname) {
 	} else {
 		Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, if_index);
 	}
-
 	Log_Info("%s: starting setup of xdp program on "
 		 "interface %s (%d)",
 		 __FUNCTION__, ifname, if_index);
 
 	err = xsk_setup_xdp_prog(if_index, &fd);
+	Log_Info("Error value: %d", err);
 	if (err) {
 		Log_Error("%s: setup of xdp program failed, "
 			  "returned: %d",
@@ -59,7 +59,7 @@ int Load_bpf_send_xsk_map(char *ifname) {
 			 __FUNCTION__, ifname, if_index, fd);
 		return fd;
 	}
-
+	Log_Info("FD value: %d", fd);
 	return -1;
 }
 
@@ -136,6 +136,7 @@ int Configure_busy_poll(int fd, int busy_timeout, int busy_budget) {
 int Clean_bpf(char *ifname) {
 	int if_index, err;
 	int fd = -1;
+	struct xdp_multiprog *mp = NULL;
 
 	Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname);
 
@@ -150,19 +151,19 @@ int Clean_bpf(char *ifname) {
 	Log_Info("%s: starting removal of xdp program on interface %s (%d)", __FUNCTION__, ifname,
 		 if_index);
 
-	err = bpf_set_link_xdp_fd(if_index, fd, XDP_FLAGS_UPDATE_IF_NOEXIST);
+	mp = xdp_multiprog__get_from_ifindex(if_index);
+	if (!mp) {
+		Log_Error("%s: unable to receive correct multi_prog reference : %s", __FUNCTION__,
+			  mp);
+		return -1;
+	}
+
+	err = xdp_multiprog__detach(mp);
 	if (err) {
-		if (err == EBUSY_CODE_WARNING) {
-			// unloading of XDP program found to return EBUSY error of -16 on certain
-			// host libbpf versions. doesn't break functionality and this problem is
-			// being investigated.
-			Log_Warning("%s: Removal of xdp program is reporting error code: (%d)",
-				    __FUNCTION__, err);
-		} else {
-			Log_Error("%s: Removal of xdp program failed, returned: (%d)", __FUNCTION__,
-				  err);
-			return 1;
-		}
+		Log_Error("%s: Removal of xdp program failed, returned: "
+			  "returned: %d",
+			  __FUNCTION__, err);
+		return -1;
 	}
 
 	Log_Info("%s: removed xdp program from interface %s (%d)", __FUNCTION__, ifname, if_index);
@@ -173,6 +174,7 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 	int prog_fd = -1, err, ifindex;
 	char *filename = "/afxdp/xdp_pass.o";
 	struct bpf_object *obj;
+	struct xdp_program *prog;
 	__u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_DRV_MODE;
 
 	Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname);
@@ -190,17 +192,22 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 		 __FUNCTION__, ifname, ifindex);
 
 	/* Load the BPF program */
-	err = bpf_prog_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
-	if (err < 0) {
-		Log_Error("%s: Couldn't load BPF-OBJ file(%s)\n", __FUNCTION__, filename);
-		return -1;
+	prog = xdp_program__open_file(filename, NULL, NULL);
+	err = libxdp_get_error(prog);
+	if (err) {
+		libxdp_strerror(err, "Couldn’t load XDP program",
+				sizeof("Couldn’t load XDP program"));
+		Log_Error("%s: Couldn’t load XDP program\n", __FUNCTION__, filename);
+		return err;
 	}
 
 	/* Attach the program to the interface at the xdp hook */
-	err = bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags);
-	if (err < 0) {
+	err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0);
+	if (err) {
+		libxdp_strerror(err, "Couldn't attach the xdp pass program",
+				sizeof("Couldn't attach the xdp pass program"));
 		Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname);
-		return -1;
+		return err;
 	}
 
 	Log_Info("%s: xdp-pass program loaded on %s (%d)", __FUNCTION__, ifname, ifindex);

diff --git a/internal/bpf/bpfWrapper.go b/internal/bpf/bpfWrapper.go
@@ -16,10 +16,10 @@
 
 package bpf
 
-//#include <bpf/libbpf.h>
-//#include <bpf/xsk.h>
+//#include <xdp/libxdp.h>
+//#include <xdp/xsk.h>
 //#cgo CFLAGS: -I.
-//#cgo LDFLAGS: -L. -lbpf
+//#cgo LDFLAGS: -L. -lxdp -lbpf -lelf -lz
 //#include "bpfWrapper.h"
 //#include "log.h"
 import "C"

diff --git a/internal/cni/cni.go b/internal/cni/cni.go
@@ -503,4 +503,4 @@ func extractIP(result *current.Result) (string, error) {
 	err := fmt.Errorf("extractIP(): ip is an empty string")
 
 	return resultIP, err
-}
+}
diff --git a/internal/host/host.go b/internal/host/host.go
@@ -17,11 +17,12 @@ package host
 
 import (
 	"errors"
-	logging "github.com/sirupsen/logrus"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
+
+	logging "github.com/sirupsen/logrus"
 )
 
 /*
@@ -33,7 +34,7 @@ type Handler interface {
 	AllowsUnprivilegedBpf() (bool, error)
 	KernelVersion() (string, error)
 	HasEthtool() (bool, string, error)
-	HasLibbpf() (bool, []string, error)
+	HasLibxdp() (bool, []string, error)
 	HasDevlink() (bool, string, error)
 	Hostname() (string, error)
 }
@@ -73,7 +74,7 @@ func (r *handler) KernelVersion() (string, error) {
 HasLibbpf checks if the host has libbpf installed and returns a boolean.
 It also returns a string array of libbpf libraries found under /usr/lib(64)/
 */
-func (r *handler) HasLibbpf() (bool, []string, error) {
+func (r *handler) HasLibxdp() (bool, []string, error) {
 	libPaths := []string{"/usr/lib/", "/usr/lib64/"}
 	foundLibbpf := false
 	var foundLibs []string
@@ -90,7 +91,7 @@ func (r *handler) HasLibbpf() (bool, []string, error) {
 		}
 
 		for _, file := range files {
-			if strings.Contains(file.Name(), "libbpf.so") {
+			if strings.Contains(file.Name(), "libxdp.so") {
 				foundLibbpf = true
 				foundLibs = append(foundLibs, path+file.Name())
 			}
@@ -254,4 +255,3 @@ func GivePermissions(filepath, uid, permissions string) error {
 	logging.Infof("Socket access granted to UID %s", uid)
 	return nil
 }
-