diff --git a/.github/workflows/public-ci.yml b/.github/workflows/public-ci.yml index ecc975a6..bc6f0726 100644 --- a/.github/workflows/public-ci.yml +++ b/.github/workflows/public-ci.yml @@ -38,8 +38,23 @@ jobs: with: go-version: 1.19 - - name: Install dependencies - run: sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib + - name: Install libbpf and libxdp + run: | + sudo apt update + sudo apt install -y wget build-essential golang + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb + sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb + sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y clang + sudo apt install -y llvm + sudo apt install -y gcc-multilib + + - name: Run build run: make build @@ -54,8 +69,18 @@ jobs: with: go-version: 1.19 - - name: Install dependencies - run: sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib + - name: Install libbpf and libxdp + run: | + sudo apt update + sudo apt install -y wget build-essential golang + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb + sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb + sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb - name: Install staticcheck run: go install honnef.co/go/tools/cmd/staticcheck@latest @@ -73,8 +98,21 @@ jobs: with: go-version: 1.19 - - name: Install dependencies - run: sudo apt-get update && sudo apt install libbpf-dev clang llvm gcc-multilib + - name: Install libbpf and libxdp + run: | + sudo apt update + sudo apt install -y wget build-essential golang + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf1_1.1.0-1_amd64.deb + sudo wget http://security.ubuntu.com/ubuntu/pool/main/libb/libbpf/libbpf-dev_1.1.0-1_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/main/x/xdp-tools/libxdp-dev_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libbpf1_1.1.0-1_amd64.deb + sudo apt install -y ./libbpf-dev_1.1.0-1_amd64.deb + sudo apt install -y ./libxdp1_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y ./libxdp-dev_1.3.0-2ubuntu2_amd64.deb + sudo apt install -y clang + sudo apt install -y llvm + sudo apt install -y gcc-multilib - name: unit-tests run: make test diff --git a/Makefile b/Makefile index 64372882..4650c7e1 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ format: clangformat buildc: @echo "****** Build BPF ******" @echo - gcc ./internal/bpf/bpfWrapper.c -lbpf -c -o ./internal/bpf/bpfWrapper.o + gcc ./internal/bpf/bpfWrapper.c -lxdp -c -o ./internal/bpf/bpfWrapper.o ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o &> /dev/null @echo "****** Build xdp_pass ******" make -C ./internal/bpf/xdp-pass/ @@ -58,7 +58,7 @@ builddp: buildc buildcni: buildc @echo "****** Build CNI ******" @echo - go build -o ./bin/afxdp ./cmd/cni + go build -ldflags="-extldflags=-static" -tags netgo -o ./bin/afxdp ./cmd/cni @echo @echo @@ -68,24 +68,24 @@ build: builddp buildcni docker: ## Build docker image @echo "****** Docker Image ******" @echo - docker build -t afxdp-device-plugin -f images/amd64.dockerfile . + docker build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile . @echo @echo podman: ## Build podman image @echo "****** Podman Image ******" @echo - podman build -t afxdp-device-plugin -f images/amd64.dockerfile . + podman build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile . @echo @echo image: - if $(MAKE) podman; then \ - echo "Podman build succeeded"; \ - else \ - echo "Podman build failed, trying docker.."; \ + # if $(MAKE) podman; then \ + # echo "Podman build succeeded"; \ + # else \ + # echo "Podman build failed, trying docker.."; \ $(MAKE) docker; \ - fi + # fi undeploy: ## Undeploy the Deamonset @echo "****** Stop Daemonset ******" diff --git a/cmd/deviceplugin/main.go b/cmd/deviceplugin/main.go index 2428e295..289a21c2 100644 --- a/cmd/deviceplugin/main.go +++ b/cmd/deviceplugin/main.go @@ -185,21 +185,23 @@ func configureLogging(cfg deviceplugin.PluginConfig) error { // On each Kind node // Create a bridge afxdp-kind-br // Create 4 vethpairs starting at veth6 -// +===============+ -// | afxdp-kind-br | -// | +---------| +---------+ -// | | veth7 | <=====> | veth6 | -// | +---------| +---------+ -// | +---------| +---------+ -// | | veth9 | <=====> | veth8 | -// | +---------| +---------+ -// | +---------| +---------+ -// | | veth11 | <=====> | veth10 | -// | +---------| +---------+ -// | +---------| +---------+ -// | | veth13 | <=====> | veth12 | -// | +---------| +---------+ -// +===============+ +// +// +===============+ +// | afxdp-kind-br | +// | +---------| +---------+ +// | | veth7 | <=====> | veth6 | +// | +---------| +---------+ +// | +---------| +---------+ +// | | veth9 | <=====> | veth8 | +// | +---------| +---------+ +// | +---------| +---------+ +// | | veth11 | <=====> | veth10 | +// | +---------| +---------+ +// | +---------| +---------+ +// | | veth13 | <=====> | veth12 | +// | +---------| +---------+ +// +===============+ +// // The "even" veth of the pair will be added to the device plugin resource pool. // and plumbed to the Pod. func configureKindSecondaryNetwork() error { @@ -249,7 +251,7 @@ func checkHost(host host.Handler) (bool, error) { // libbpf logging.Debugf("Checking host for Libbpf") - bpfInstalled, libs, err := host.HasLibbpf() + bpfInstalled, libs, err := host.HasLibxdp() if err != nil { err := fmt.Errorf("libbpf not found on host") return false, err diff --git a/images/amd64.dockerfile b/images/amd64.dockerfile index 7585e42b..10b683ec 100644 --- a/images/amd64.dockerfile +++ b/images/amd64.dockerfile @@ -12,24 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.20@sha256:52921e63cc544c79c111db1d8461d8ab9070992d9c636e1573176642690c14b5 as cnibuilder +FROM golang:1.20@sha256:efe38cb419e2b2012f66d1782d2efe2fd8884c71d9f342581e1697ba9047b5f8 as cnibuilder COPY . /usr/src/afxdp_k8s_plugins WORKDIR /usr/src/afxdp_k8s_plugins -RUN apt-get update && apt-get -y install --no-install-recommends libbpf-dev=1:0.3-2 \ - && apt-get -y install --no-install-recommends clang=1:11.0-51+nmu5 llvm=1:11.0-51+nmu5 gcc-multilib=4:10.2.1-1 \ - && make buildcni +RUN apt-get update \ +&& apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 \ +&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6 \ +&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 \ +&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 \ +&& make buildcni -FROM golang:1.20-alpine@sha256:87d0a3309b34e2ca732efd69fb899d3c420d3382370fd6e7e6d2cb5c930f27f9 as dpbuilder +FROM golang:1.20-alpine@sha256:ebceb16dc094769b6e2a393d51e0417c19084ba20eb8967fb3f7675c32b45774 as dpbuilder COPY . /usr/src/afxdp_k8s_plugins WORKDIR /usr/src/afxdp_k8s_plugins -RUN apk add --no-cache build-base~=0.5 libbsd-dev~=0.11 \ - && apk add --no-cache libbpf-dev~=0.5 --repository=https://dl-cdn.alpinelinux.org/alpine/v3.15/community \ - && apk add --no-cache llvm~=15.0.7-r0 clang~=15.0.7-r0 \ - && make builddp +RUN apk add --no-cache build-base~=0.5-r3 \ +&& apk add --no-cache libbsd-dev~=0.11.7 \ +&& apk add --no-cache libxdp-dev~=1.2.10-r0 \ +&& apk add --no-cache llvm15~=15.0.7-r0 \ +&& apk add --no-cache clang15~=15.0.7-r0 \ +&& make builddp -FROM amd64/alpine:3.17@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 -RUN apk --no-cache -U add iproute2-rdma~=6.0 acl~=2.3 \ - && apk --no-cache -U add libbpf~=0.5 --repository=http://dl-cdn.alpinelinux.org/alpine/v3.15/community +FROM amd64/alpine:3.18@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70 +RUN apk --no-cache -U add iproute2-rdma~=6.3.0-r0 acl~=2.3 \ + && apk add --no-cache libxdp~=1.2.10-r0 COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh diff --git a/internal/bpf/bpfWrapper.c b/internal/bpf/bpfWrapper.c index eeb0fdca..c427a28a 100644 --- a/internal/bpf/bpfWrapper.c +++ b/internal/bpf/bpfWrapper.c @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#include // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd #include // for XDP_FLAGS_DRV_MODE #include // for if_nametoindex +#include +#include // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd #include "bpfWrapper.h" #include "log.h" @@ -40,12 +40,12 @@ int Load_bpf_send_xsk_map(char *ifname) { } else { Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, if_index); } - Log_Info("%s: starting setup of xdp program on " "interface %s (%d)", __FUNCTION__, ifname, if_index); err = xsk_setup_xdp_prog(if_index, &fd); + Log_Info("Error value: %d", err); if (err) { Log_Error("%s: setup of xdp program failed, " "returned: %d", @@ -59,7 +59,7 @@ int Load_bpf_send_xsk_map(char *ifname) { __FUNCTION__, ifname, if_index, fd); return fd; } - + Log_Info("FD value: %d", fd); return -1; } @@ -136,6 +136,7 @@ int Configure_busy_poll(int fd, int busy_timeout, int busy_budget) { int Clean_bpf(char *ifname) { int if_index, err; int fd = -1; + struct xdp_multiprog *mp = NULL; Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname); @@ -150,19 +151,19 @@ int Clean_bpf(char *ifname) { Log_Info("%s: starting removal of xdp program on interface %s (%d)", __FUNCTION__, ifname, if_index); - err = bpf_set_link_xdp_fd(if_index, fd, XDP_FLAGS_UPDATE_IF_NOEXIST); + mp = xdp_multiprog__get_from_ifindex(if_index); + if (!mp) { + Log_Error("%s: unable to receive correct multi_prog reference : %s", __FUNCTION__, + mp); + return -1; + } + + err = xdp_multiprog__detach(mp); if (err) { - if (err == EBUSY_CODE_WARNING) { - // unloading of XDP program found to return EBUSY error of -16 on certain - // host libbpf versions. doesn't break functionality and this problem is - // being investigated. - Log_Warning("%s: Removal of xdp program is reporting error code: (%d)", - __FUNCTION__, err); - } else { - Log_Error("%s: Removal of xdp program failed, returned: (%d)", __FUNCTION__, - err); - return 1; - } + Log_Error("%s: Removal of xdp program failed, returned: " + "returned: %d", + __FUNCTION__, err); + return -1; } Log_Info("%s: removed xdp program from interface %s (%d)", __FUNCTION__, ifname, if_index); @@ -173,6 +174,7 @@ int Load_attach_bpf_xdp_pass(char *ifname) { int prog_fd = -1, err, ifindex; char *filename = "/afxdp/xdp_pass.o"; struct bpf_object *obj; + struct xdp_program *prog; __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_DRV_MODE; Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname); @@ -190,17 +192,22 @@ int Load_attach_bpf_xdp_pass(char *ifname) { __FUNCTION__, ifname, ifindex); /* Load the BPF program */ - err = bpf_prog_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd); - if (err < 0) { - Log_Error("%s: Couldn't load BPF-OBJ file(%s)\n", __FUNCTION__, filename); - return -1; + prog = xdp_program__open_file(filename, NULL, NULL); + err = libxdp_get_error(prog); + if (err) { + libxdp_strerror(err, "Couldn’t load XDP program", + sizeof("Couldn’t load XDP program")); + Log_Error("%s: Couldn’t load XDP program\n", __FUNCTION__, filename); + return err; } /* Attach the program to the interface at the xdp hook */ - err = bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags); - if (err < 0) { + err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0); + if (err) { + libxdp_strerror(err, "Couldn't attach the xdp pass program", + sizeof("Couldn't attach the xdp pass program")); Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname); - return -1; + return err; } Log_Info("%s: xdp-pass program loaded on %s (%d)", __FUNCTION__, ifname, ifindex); diff --git a/internal/bpf/bpfWrapper.go b/internal/bpf/bpfWrapper.go index 2934d379..ad491943 100755 --- a/internal/bpf/bpfWrapper.go +++ b/internal/bpf/bpfWrapper.go @@ -16,10 +16,10 @@ package bpf -//#include -//#include +//#include +//#include //#cgo CFLAGS: -I. -//#cgo LDFLAGS: -L. -lbpf +//#cgo LDFLAGS: -L. -lxdp -lbpf -lelf -lz //#include "bpfWrapper.h" //#include "log.h" import "C" diff --git a/internal/cni/cni.go b/internal/cni/cni.go index cc162efe..ca27e15b 100644 --- a/internal/cni/cni.go +++ b/internal/cni/cni.go @@ -503,4 +503,4 @@ func extractIP(result *current.Result) (string, error) { err := fmt.Errorf("extractIP(): ip is an empty string") return resultIP, err -} +} \ No newline at end of file diff --git a/internal/host/host.go b/internal/host/host.go index 50776588..af03e536 100644 --- a/internal/host/host.go +++ b/internal/host/host.go @@ -17,11 +17,12 @@ package host import ( "errors" - logging "github.com/sirupsen/logrus" "io/ioutil" "os" "os/exec" "strings" + + logging "github.com/sirupsen/logrus" ) /* @@ -33,7 +34,7 @@ type Handler interface { AllowsUnprivilegedBpf() (bool, error) KernelVersion() (string, error) HasEthtool() (bool, string, error) - HasLibbpf() (bool, []string, error) + HasLibxdp() (bool, []string, error) HasDevlink() (bool, string, error) Hostname() (string, error) } @@ -73,7 +74,7 @@ func (r *handler) KernelVersion() (string, error) { HasLibbpf checks if the host has libbpf installed and returns a boolean. It also returns a string array of libbpf libraries found under /usr/lib(64)/ */ -func (r *handler) HasLibbpf() (bool, []string, error) { +func (r *handler) HasLibxdp() (bool, []string, error) { libPaths := []string{"/usr/lib/", "/usr/lib64/"} foundLibbpf := false var foundLibs []string @@ -90,7 +91,7 @@ func (r *handler) HasLibbpf() (bool, []string, error) { } for _, file := range files { - if strings.Contains(file.Name(), "libbpf.so") { + if strings.Contains(file.Name(), "libxdp.so") { foundLibbpf = true foundLibs = append(foundLibs, path+file.Name()) } @@ -254,4 +255,3 @@ func GivePermissions(filepath, uid, permissions string) error { logging.Infof("Socket access granted to UID %s", uid) return nil } - diff --git a/internal/host/host_fake.go b/internal/host/host_fake.go index 874a4c3a..d3276558 100644 --- a/internal/host/host_fake.go +++ b/internal/host/host_fake.go @@ -67,7 +67,7 @@ func (r *fakeHandler) HasEthtool() (bool, string, error) { HasLibbpf checks if the host has libbpf installed and returns a boolean. In this FakeHandler it returns a dummy value. */ -func (r *fakeHandler) HasLibbpf() (bool, []string, error) { +func (r *fakeHandler) HasLibxdp() (bool, []string, error) { return true, nil, nil }