Feat kind support map pinning v3 (#67)

* bpf: map pinning support

* cleanup: comments in bpfWrapper.c

* dpcnisyncer: basic signalling from CNI to DP

* dpcnisyncer: delete BPFFS signalled from CNI

* pool-manager: dp-cni-syncer cleanup

* SyncServer: one server for all BPF map managers

* pool-manager: remove map cleanup

* dpcnisyncer: add clean up

* Makefile: Add missing copyright tag

* fixup: unit tests

* fixup: static analysis

* linter: add directive to ignore generated code

* fixup: fuzzing test

* Add missing protobuf dependency

* syncer: explicit configuration

* plugin: updates after rebase

* bpfwrapper: use libxdp for redirect program

* cleanup: dead/commented code

* linter error: run clang format
---------

Signed-off-by: Maryam Tahhan <[email protected]>

Makefile

@@ -27,6 +27,7 @@ clangformat:
 	@echo
 	-clang-format -i -style=file internal/bpf/*.c internal/bpf/*.h
 	-clang-format -i -style=file internal/bpf/xdp-pass/*.c
+	-clang-format -i -style=file internal/bpf/xdp-afxdp-redirect/*.c
 	@echo
 	@echo
 
@@ -37,18 +38,22 @@ format: clangformat
 	@echo
 	@echo
 
+buildxdp:
+	@echo "******     Build xdp_pass     ******"
+	make -C ./internal/bpf/xdp-pass/
+	@echo "******     Build xdp_afxdp_redirect     ******"
+	make -C ./internal/bpf/xdp-afxdp-redirect/
+	@echo
+
 buildc:
 	@echo "******     Build BPF     ******"
 	@echo
 	gcc ./internal/bpf/bpfWrapper.c -lxdp -c -o ./internal/bpf/bpfWrapper.o
 	ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o  &> /dev/null
-	@echo "******     Build xdp_pass     ******"
-	make -C ./internal/bpf/xdp-pass/
-	@echo
 	@echo
 	@echo
 
-builddp: buildc
+builddp: buildc buildxdp
 	@echo "******     Build DP      ******"
 	@echo
 	go build -o ./bin/afxdp-dp ./cmd/deviceplugin

README.md

@@ -407,6 +407,10 @@ Note: User 0 does not imply that the pod needs to be privileged.
 
 UdsServerDisable is a Boolean configuration. If set to true, devices in this pool will not have the BPF app loaded onto the netdev. This means no UDS server is spun up when a device is allocated to a pod. By default, this is set to false.
 
+#### BpfMapPinningEnable
+
+BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration.
+
 #### UdsTimeout
 
 UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds.

cmd/deviceplugin/main.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/intel/afxdp-plugins-for-kubernetes/constants"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/deviceplugin"
+	"github.com/intel/afxdp-plugins-for-kubernetes/internal/dpcnisyncerserver"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/host"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/logformats"
 	"github.com/intel/afxdp-plugins-for-kubernetes/internal/networking"
@@ -97,9 +98,16 @@ func main() {
 	}
 	logging.Infof("Host meets requirements")
 
+	//START THE SYNCER SERVER TODO CHECK BPF MAP
+	dpCniSyncerServer, err := dpcnisyncerserver.NewSyncerServer()
+	if err != nil {
+		logging.Errorf("Error creating the DpCniSyncerServer")
+	}
+	logging.Debugf("DP<=>CNI grpc Syncer started")
+
 	// pool configs
 	logging.Infof("Getting device pools")
-	poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler)
+	poolConfigs, err := deviceplugin.GetPoolConfigs(configFile, netHandler, hostHandler, dpCniSyncerServer)
 	if err != nil {
 		logging.Warningf("Error getting device pools: %v", err)
 		exit(constants.Plugins.DevicePlugin.ExitPoolError)

constants/constants.go

@@ -82,6 +82,10 @@ var (
 	udsSockDir    = "/tmp/afxdp_dp/"  // host location where we place our uds sockets. If changing location remember to update daemonset mount point
 	udsPodPath    = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
 
+	/* BPF*/
+	bpfMapPodPath = "/tmp/xsks_map"
+	xsk_map       = "/xsks_map"
+
 	udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
 
 	/* Handshake*/
@@ -129,6 +133,8 @@ var (
 	DeviceFile deviceFile
 	/* DeviceFile contains constants related to the devicefile */
 	EthtoolFilter ethtoolFilter
+	/* Bpf contains constants related to the BPF Map pinning */
+	Bpf bpf
 )
 
 type cni struct {
@@ -213,6 +219,11 @@ type uds struct {
 	Handshake   handshake
 }
 
+type bpf struct {
+	BpfMapPodPath string
+	Xsk_map       string
+}
+
 type handshake struct {
 	Version             string
 	RequestVersion      string
@@ -334,6 +345,11 @@ func init() {
 		},
 	}
 
+	Bpf = bpf{
+		BpfMapPodPath: bpfMapPodPath,
+		Xsk_map:       xsk_map,
+	}
+
 	EthtoolFilter = ethtoolFilter{
 		EthtoolFilterRegex: ethtoolFilterRegex,
 	}

deployments/daemonset-kind.yaml

@@ -13,7 +13,8 @@ data:
           {
             "name":"myPool",
             "mode":"primary",
-            "UdsServerDisable": false,
+            "UdsServerDisable": true,
+            "BpfMapPinningEnable": true,
             "drivers":[
               {
                 "name":"veth"
@@ -72,6 +73,8 @@ spec:
           volumeMounts:
             - name: unixsock
               mountPath: /tmp/afxdp_dp/
+            - name: bpfmappinning
+              mountPath: /var/run/afxdp_dp/
               mountPropagation: Bidirectional
             - name: devicesock
               mountPath: /var/lib/kubelet/device-plugins/
@@ -87,6 +90,9 @@ spec:
         - name: unixsock
           hostPath:
             path: /tmp/afxdp_dp/
+        - name: bpfmappinning
+          hostPath:
+            path: /var/run/afxdp_dp/
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/device-plugins/

examples/cndp-0-0.yaml

@@ -0,0 +1,26 @@
+# A working example of BPF MAP PINNING
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cndp-0-0
+  annotations:
+    k8s.v1.cni.cncf.io/networks: afxdp-network
+spec:
+  containers:
+    - name: cndp-0
+      command: ["/bin/bash"]
+      args: ["-c", "./jsonc_gen.sh -kp ; cndpfwd -c config.jsonc lb;"]
+      image:  quay.io/mtahhan/cndp-map-pinning:latest
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+          #capabilities:
+          #add:
+          #  - NET_RAW
+          #  - IPC_LOCK
+          #  - BPF
+      resources:
+        requests:
+          afxdp/myPool: '1'
+        limits:
+          afxdp/myPool: '1'

examples/nad_with_syncer.yaml

@@ -0,0 +1,27 @@
+# WARNING: This is an example definition only. Remove all comments before use.
+
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: afxdp-network                                        # Name of this network, pods will request this network by name
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: afxdp/myPool            # Needs to match the device plugin pool name / resource type
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "afxdp",                                       # CNI binary, leave as afxdp
+      "mode": "primary",                                     # CNI mode setting (required)
+      "logFile": "afxdp-cni.log",                            # CNI log file location (optional)
+      "logLevel": "debug",                                   # CNI logging level (optional)
+      "dpSyncer": true,                                      # Sync with Device Plugin over gRPC MUST BE SET for bpf map pinning
+      "ipam": {                                              # CNI IPAM plugin and associated config (optional)
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.220",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'

go.mod

@@ -6,17 +6,20 @@ require (
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.1.1
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
+	github.com/golang/protobuf v1.5.3
 	github.com/google/gofuzz v1.1.0
 	github.com/google/uuid v1.3.0
 	github.com/intel/afxdp-plugins-for-kubernetes/pkg/goclient v0.0.0
 	github.com/intel/afxdp-plugins-for-kubernetes/pkg/subfunctions v0.0.0
+	github.com/moby/sys/mount v0.3.3
 	github.com/pkg/errors v0.9.1
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
 	golang.org/x/net v0.17.0
 	google.golang.org/grpc v1.53.0
+	google.golang.org/protobuf v1.30.0 // indirect
 	gotest.tools v2.2.0+incompatible
 	k8s.io/apimachinery v0.25.2
 	k8s.io/kubelet v0.25.2

go.sum

@@ -735,8 +735,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -894,8 +895,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
+github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
@@ -1774,8 +1779,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

images/amd64.dockerfile

@@ -17,8 +17,8 @@ COPY . /usr/src/afxdp_k8s_plugins
 WORKDIR /usr/src/afxdp_k8s_plugins
 RUN apt-get update \
 && apt-get -y install --no-install-recommends libxdp-dev=1.3.1-1 \
-&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.6  \
-&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.6 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends clang=1:14.0-55.7~deb12u1 \
+&& apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends llvm=1:14.0-55.7~deb12u1 \
 && apt-get -y install -o APT::Keep-Downloaded-Packages=false --no-install-recommends gcc-multilib=4:12.2.0-3 \
 && make buildcni
 
@@ -39,4 +39,5 @@ COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-pass/xdp_pass.o /afxdp/xdp_pass.o
+COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.o /afxdp/xdp_afxdp_redirect.o
 ENTRYPOINT ["/afxdp/entrypoint.sh"]

internal/bpf/bpfWrapper.c

@@ -5,16 +5,21 @@
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *	 http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+#include <errno.h>
+#include <fcntl.h>
 #include <linux/if_link.h> // for XDP_FLAGS_DRV_MODE
 #include <net/if.h>	   // for if_nametoindex
+#include <sys/stat.h>
+#include <unistd.h>
 #include <xdp/libxdp.h>
 #include <xdp/xsk.h> // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd
 
@@ -183,8 +188,13 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 	if (!ifindex) {
 		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
 		return -1;
-	} else {
-		Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+	}
+	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+
+	if (access(filename, O_RDONLY) < 0) {
+		Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename,
+			  strerror(errno));
+		return err;
 	}
 
 	Log_Info("%s: starting setup of xdp-pass program on "
@@ -214,3 +224,52 @@ int Load_attach_bpf_xdp_pass(char *ifname) {
 
 	return 0;
 }
+
+int Load_bpf_pin_xsk_map(char *ifname, char *pin_path) {
+	struct bpf_object *obj;
+	struct bpf_program *prog;
+	struct bpf_link *link;
+	int ifindex, map_fd = -1;
+	int err;
+	const char *prog_name = "xdp_afxdp_redirect";
+	char *filename = "/afxdp/xdp_afxdp_redirect.o";
+	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, bpf_opts, .pin_root_path = pin_path);
+
+	ifindex = if_nametoindex(ifname);
+	if (!ifindex) {
+		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
+		return -1;
+	}
+	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+
+	if (access(filename, O_RDONLY) < 0) {
+		Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename,
+			  strerror(errno));
+		return err;
+	}
+
+	Log_Info("%s: starting setup of xdp-redirect program on "
+		 "interface %s (%d)",
+		 __FUNCTION__, ifname, ifindex);
+
+	/* Load the BPF program */
+	prog = xdp_program__open_file(filename, NULL, NULL);
+	err = libxdp_get_error(prog);
+	if (err) {
+		libxdp_strerror(err, "Couldn’t load XDP program",
+				sizeof("Couldn’t load XDP program"));
+		Log_Error("%s: Couldn’t load XDP program\n", __FUNCTION__, filename);
+		return err;
+	}
+
+	/* Attach the program to the interface at the xdp hook */
+	err = xdp_program__attach(prog, ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, 0);
+	if (err) {
+		libxdp_strerror(err, "Couldn't attach the xdp pass program",
+				sizeof("Couldn't attach the xdp pass program"));
+		Log_Error("%s: Couldn't attach the XDP PASS PROGRAM TO %s\n", __FUNCTION__, ifname);
+		return err;
+	}
+
+	return 0;
+}

internal/bpf/bpfWrapper.go

@@ -39,6 +39,7 @@ type Handler interface {
 	LoadBpfSendXskMap(ifname string) (int, error)
 	LoadAttachBpfXdpPass(ifname string) error
 	ConfigureBusyPoll(fd int, busyTimeout int, busyBudget int) error
+	LoadBpfPinXskMap(ifname, pin_path string) error
 	Cleanbpf(ifname string) error
 }
 
@@ -80,6 +81,19 @@ func (r *handler) LoadAttachBpfXdpPass(ifname string) error {
 	return nil
 }
 
+/*
+LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_send_xsk_map
+*/
+func (r *handler) LoadBpfPinXskMap(ifname, pin_path string) error {
+	err := int(C.Load_bpf_pin_xsk_map(C.CString(ifname), C.CString(pin_path)))
+
+	if err < 0 {
+		return errors.New("error loading BPF program onto interface")
+	}
+
+	return nil
+}
+
 /*
 ConfigureBusyPoll is the GoLang wrapper for the C function Configure_busy_poll
 */