diff --git a/Makefile b/Makefile index 64372882..8887fd55 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ format: clangformat buildc: @echo "****** Build BPF ******" @echo - gcc ./internal/bpf/bpfWrapper.c -lbpf -c -o ./internal/bpf/bpfWrapper.o + gcc ./internal/bpf/bpfWrapper.c -lbpf -lxdp -c -o ./internal/bpf/bpfWrapper.o ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o &> /dev/null @echo "****** Build xdp_pass ******" make -C ./internal/bpf/xdp-pass/ @@ -58,7 +58,7 @@ builddp: buildc buildcni: buildc @echo "****** Build CNI ******" @echo - go build -o ./bin/afxdp ./cmd/cni + go build -ldflags="-extldflags=-static" -tags netgo -o ./bin/afxdp ./cmd/cni @echo @echo diff --git a/constants/constants.go b/constants/constants.go index 94451a6d..df87bae9 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -74,13 +74,13 @@ var ( afxdpMinimumLinux = "4.18.0" // minimum Linux version for AF_XDP support /* UDS*/ - udsMaxTimeout = 300 // maximum configurable uds timeout in seconds - udsMinTimeout = 30 // minimum (and default) uds timeout in seconds - udsMsgBufSize = 64 // uds message buffer size - udsCtlBufSize = 4 // uds control buffer size - udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET - udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point - udsPodPath = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod + udsMaxTimeout = 300 // maximum configurable uds timeout in seconds + udsMinTimeout = 0 // minimum (and default) uds timeout in seconds + udsMsgBufSize = 64 // uds message buffer size + udsCtlBufSize = 4 // uds control buffer size + udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET + udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point + udsPodPath = "/tmp/afxdp" // the uds filepath as it will appear in the end user application pod udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets diff --git a/images/amd64.dockerfile b/images/amd64.dockerfile index 7585e42b..c890db49 100644 --- a/images/amd64.dockerfile +++ b/images/amd64.dockerfile @@ -12,24 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.20@sha256:52921e63cc544c79c111db1d8461d8ab9070992d9c636e1573176642690c14b5 as cnibuilder +FROM golang:1.20 as cnibuilder COPY . /usr/src/afxdp_k8s_plugins WORKDIR /usr/src/afxdp_k8s_plugins -RUN apt-get update && apt-get -y install --no-install-recommends libbpf-dev=1:0.3-2 \ - && apt-get -y install --no-install-recommends clang=1:11.0-51+nmu5 llvm=1:11.0-51+nmu5 gcc-multilib=4:10.2.1-1 \ - && make buildcni +RUN apt-get update +RUN apt-get update && apt-get -y install --no-install-recommends libbpf-dev=1:1.1.0-1 +RUN apt-get update && apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 +RUN apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6 +RUN apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 +RUN apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 +RUN make buildcni -FROM golang:1.20-alpine@sha256:87d0a3309b34e2ca732efd69fb899d3c420d3382370fd6e7e6d2cb5c930f27f9 as dpbuilder +FROM golang:1.20-alpine as dpbuilder COPY . /usr/src/afxdp_k8s_plugins WORKDIR /usr/src/afxdp_k8s_plugins -RUN apk add --no-cache build-base~=0.5 libbsd-dev~=0.11 \ - && apk add --no-cache libbpf-dev~=0.5 --repository=https://dl-cdn.alpinelinux.org/alpine/v3.15/community \ - && apk add --no-cache llvm~=15.0.7-r0 clang~=15.0.7-r0 \ - && make builddp +RUN apk add --no-cache build-base~=0.5-r3 +RUN apk add --no-cache libbsd-dev~=0.11.7 +# TODO: Add version later +RUN apk add --no-cache libxdp-dev +RUN apk add --no-cache libbpf-dev +RUN apk add --no-cache llvm16~=16.0.6-r1 +RUN apk add --no-cache clang16~=16.0.6-r1 +RUN make builddp -FROM amd64/alpine:3.17@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501 -RUN apk --no-cache -U add iproute2-rdma~=6.0 acl~=2.3 \ - && apk --no-cache -U add libbpf~=0.5 --repository=http://dl-cdn.alpinelinux.org/alpine/v3.15/community +FROM amd64/alpine:3.18 +RUN apk --no-cache -U add iproute2-rdma~=6.3.0-r0 acl~=2.3 \ + && apk add --no-cache libbpf && apk add --no-cache libxdp COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh diff --git a/internal/bpf/bpfWrapper.c b/internal/bpf/bpfWrapper.c index eeb0fdca..cfc4de2d 100644 --- a/internal/bpf/bpfWrapper.c +++ b/internal/bpf/bpfWrapper.c @@ -14,9 +14,12 @@ * limitations under the License. */ -#include // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd +#include // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd #include // for XDP_FLAGS_DRV_MODE #include // for if_nametoindex +#include +#include +// #include #include "bpfWrapper.h" #include "log.h" @@ -30,22 +33,38 @@ int Load_bpf_send_xsk_map(char *ifname) { int fd = -1; int if_index, err; + // struct bpf_program *prog; + // struct bpf_object *obj; Log_Info("%s: disovering if_index for interface %s", __FUNCTION__, ifname); if_index = if_nametoindex(ifname); + // obj = bpf_object__open(""); if (!if_index) { Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname); return -1; } else { Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, if_index); } - + // Below requires BPF map + // bpf_map__fd() + // below feeds ^ + // bpf_object__find_map_by_name + // below feeds ^ + // bpf_object__open + + // Requres bpf object and bpf program + // bpf_program__fd(), returns prog fd + // below feeds ^ + // bpf_object__find_program_by_name() + // below feeds ^ + // bpf_object__open Log_Info("%s: starting setup of xdp program on " "interface %s (%d)", __FUNCTION__, ifname, if_index); - err = xsk_setup_xdp_prog(if_index, &fd); + err = xsk_setup_xdp_prog(if_index, &fd); + Log_Info("Error value: %d", err); if (err) { Log_Error("%s: setup of xdp program failed, " "returned: %d", @@ -59,7 +78,7 @@ int Load_bpf_send_xsk_map(char *ifname) { __FUNCTION__, ifname, if_index, fd); return fd; } - + Log_Info("FD value: %d", fd); return -1; } @@ -149,8 +168,7 @@ int Clean_bpf(char *ifname) { Log_Info("%s: starting removal of xdp program on interface %s (%d)", __FUNCTION__, ifname, if_index); - - err = bpf_set_link_xdp_fd(if_index, fd, XDP_FLAGS_UPDATE_IF_NOEXIST); + err = bpf_xdp_detach(if_index, XDP_FLAGS_UPDATE_IF_NOEXIST, NULL); if (err) { if (err == EBUSY_CODE_WARNING) { // unloading of XDP program found to return EBUSY error of -16 on certain @@ -190,18 +208,18 @@ int Load_attach_bpf_xdp_pass(char *ifname) { __FUNCTION__, ifname, ifindex); /* Load the BPF program */ - err = bpf_prog_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd); - if (err < 0) { - Log_Error("%s: Couldn't load BPF-OBJ file(%s)\n", __FUNCTION__, filename); - return -1; - } + err = bpf_xdp_query_id(ifindex, (int) xdp_flags, &prog_fd); + if (err < 0) { + Log_Error("%s: Couldn't load BPF-OBJ file(%s)\n", __FUNCTION__, filename); + return -1; + } /* Attach the program to the interface at the xdp hook */ - err = bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags); - if (err < 0) { - Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname); - return -1; - } + err = bpf_xdp_attach(ifindex, prog_fd, XDP_FLAGS_UPDATE_IF_NOEXIST, NULL); + if (err < 0) { + Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname); + return -1; + } Log_Info("%s: xdp-pass program loaded on %s (%d)", __FUNCTION__, ifname, ifindex); diff --git a/internal/bpf/bpfWrapper.go b/internal/bpf/bpfWrapper.go index 2934d379..80a7e5f9 100755 --- a/internal/bpf/bpfWrapper.go +++ b/internal/bpf/bpfWrapper.go @@ -17,9 +17,9 @@ package bpf //#include -//#include +//#include //#cgo CFLAGS: -I. -//#cgo LDFLAGS: -L. -lbpf +//#cgo LDFLAGS: -L. -lbpf -lelf -lz -lxdp //#include "bpfWrapper.h" //#include "log.h" import "C" diff --git a/internal/cni/cni.go b/internal/cni/cni.go index cc162efe..838da679 100644 --- a/internal/cni/cni.go +++ b/internal/cni/cni.go @@ -18,6 +18,11 @@ package cni import ( "encoding/json" "fmt" + "os" + "regexp" + "runtime" + "strings" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" @@ -32,10 +37,6 @@ import ( "github.com/intel/afxdp-plugins-for-kubernetes/internal/tools" logging "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "os" - "regexp" - "runtime" - "strings" ) var bpfHandler = bpf.NewHandler() @@ -347,7 +348,24 @@ func CmdDel(args *skel.CmdArgs) error { if !cfg.SkipUnloadBpf { logging.Infof("cmdDel(): removing BPF program from device") - if err := bpfHandler.Cleanbpf(cfg.Device); err != nil { + + device, err := netlink.LinkByName(cfg.Device) + if err != nil { + err = fmt.Errorf("cmdDel(): failed to find device %q in containerNS: %w", cfg.Device, err) + logging.Errorf(err.Error()) + + return err + } + logging.Infof("cmdDel(): setting link to down state and attached to false") + device.Attrs().Xdp.Attached = false + if err := netlink.LinkSetDown(device); err != nil { + err = fmt.Errorf("cmdDel(): error setting device to down state: %w", err) + logging.Errorf(err.Error()) + + return err + } + logging.Infof("cmdDel():link is set to down state and attached is false") + if err := netlink.LinkSetXdpFd(device, -1); err != nil { err = fmt.Errorf("cmdDel(): error removing BPF program from device: %w", err) logging.Errorf(err.Error())