From 7a71b1d1edf653f31b6f88c63a89224f4c7f77fc Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Fri, 3 Nov 2023 15:09:12 +0000 Subject: [PATCH] Readme: cleanup and update for bpfd v0.3.0 Signed-off-by: Maryam Tahhan --- Makefile | 7 ++-- README.md | 60 +++++++++++++++++++++----------- examples/nad_with_syncer.yaml | 18 +++++----- go.mod | 1 - go.sum | 3 -- internal/bpf/xdp-pass/xdp_pass.c | 2 +- 6 files changed, 50 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index a50c16cf..43cdb9db 100644 --- a/Makefile +++ b/Makefile @@ -245,7 +245,6 @@ kind-load-images: ## Load the image on the kind cluster @echo @echo -# make KIND_CLUSTER_NAME=bpfd-deployment IMAGE=quay.io/mtahhan/cndp-map-pinning kind-load-custom-image .PHONY: kind-load-custom-image kind-load-custom-image: ## Load the image on the kind cluster @echo "****** Loading ${IMAGE} ******" @@ -273,17 +272,15 @@ kind-deploy: kind-undeploy kind-load-images ## Deploy the Deamonset and CNI in K @echo @echo -# make KIND_CLUSTER_NAME=bpfd-deployment kind-undeploy-bpfd -kind-undeploy-bpfd: ## Undeploy the Kind Deamonset +kind-undeploy-bpfd: ## Undeploy the Kind Deamonset in a bpfd cluster @echo "****** Stop Daemonset ******" @echo kubectl delete -f ./deployments/daemonset-kind-bpfd.yaml --ignore-not-found=true @echo @echo -# make KIND_CLUSTER_NAME=bpfd-deployment kind-deploy-bpfd .PHONY: kind-deploy-bpfd -kind-deploy-bpfd: kind-undeploy-bpfd kind-load-images ## Deploy the Deamonset and CNI in the Kind +kind-deploy-bpfd: kind-undeploy-bpfd kind-load-images ## Deploy the Kind Deamonset in a bpfd cluster @echo "****** Deploy Daemonset ******" @echo kubectl create -f ./deployments/daemonset-kind-bpfd.yaml diff --git a/README.md b/README.md index cf809a78..c50606a6 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ The following prerequisites are required to run the plugins: - To use a Kind deployment. - [Kind quickstart guide](https://kind.sigs.k8s.io/docs/user/quick-start/) - Tested with Kind version 0.18.0 - - **Note:** For kind, please also install [**kubeadm**](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) + - **Note:** For Kind, please also install [**kubeadm**](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) + - **Note:** For Kind, please use docker to build images. ### Development Prerequisites @@ -411,10 +412,6 @@ UdsServerDisable is a Boolean configuration. If set to true, devices in this poo BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration. -#### BpdClientEnable - -TODO, remember to disable local BPF bits if this is enabled... - #### UdsTimeout UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds. @@ -506,6 +503,14 @@ The example below shows a config including log settings. } ``` +### Ethtool Filters +Ethtool filters can be applied to devices during CNI runtime. This setting can only be assigned to devices in a `primary` mode pool from the networkAttachmentDefinition file. +The EthtoolCmds field is an array of strings. These strings should be formatted exactly as if setting Ethtool filters where manually from the command line. Some Ethtool filters require the device name or the IP address and in these instances, the user can substitute these with `-device-` and `-ip-` respectively. The plugins will apply the filters with the correct name and IP address when they become known during pod creation. + +From the [examples/network-attachment-definition.yaml](./examples/network-attachment-definition.yaml) file: **ethtoolCmds** field has two filters configured. This means the filters `ethtool -X equal 5 start 3` and `ethtool --config-ntuple -device- flow-type udp4 dst-ip action` will be configured on all devices as they are being attached to the AF_XDP pods. The plugins will substitute `` and `` accordingly. + +*Note: When setting ethtool commands in the **ethtoolCmds** field, the 'ethtool' prefix must be removed. + ### Kind Cluster The kindCluster flag is used to indicate if this is a physical cluster or a Kind cluster. @@ -532,24 +537,36 @@ The kindCluster flag is used to indicate if this is a physical cluster or a Kind } ``` -### Ethtool Filters -Ethtool filters can be applied to devices during CNI runtime. This setting can only be assigned to devices in a `primary` mode pool from the networkAttachmentDefinition file. -The EthtoolCmds field is an array of strings. These strings should be formatted exactly as if setting Ethtool filters where manually from the command line. Some Ethtool filters require the device name or the IP address and in these instances, the user can substitute these with `-device-` and `-ip-` respectively. The plugins will apply the filters with the correct name and IP address when they become known during pod creation. -From the [examples/network-attachment-definition.yaml](./examples/network-attachment-definition.yaml) file: **ethtoolCmds** field has two filters configured. This means the filters `ethtool -X equal 5 start 3` and `ethtool --config-ntuple -device- flow-type udp4 dst-ip action` will be configured on all devices as they are being attached to the AF_XDP pods. The plugins will substitute `` and `` accordingly. +#### bpfdClientEnable -*Note: When setting ethtool commands in the **ethtoolCmds** field, the 'ethtool' prefix must be removed. +bpfdClientEnable is a Boolean configuration. If set to true, will use bpfd to load bpf programs onto the netdevs allocated to the pod from the resource pool. By default, this is set to false. +Should set BpfMapPinningEnable to false and UdsServerDisable to true when using this configuration. -### bpfd Client +#### bpfd bpfd is a system daemon aimed at simplifying the deployment and management of eBPF programs. Integrating the AF_XDP device plugin with bpfd means delegating the responsibility of loading and unloading the eBPF programs for AF_XDP applications to the bpfd daemonset running on each node. This means that instead of AF_XDP DP loading/unloading a BPF program, it will create/delete -an [XdpProgram CRD](https://pkg.go.dev/github.com/bpfd-dev/bpfd@v0.2.1/bpfd-operator/apis/v1alpha1#XdpProgram) via the -[bpfd-operator APIs](https://pkg.go.dev/github.com/bpfd-dev/bpfd@v0.2.1/bpfd-operator/apis/v1alpha1). +an [XdpProgram CRD](https://pkg.go.dev/github.com/bpfd-dev/bpfd@v0.3.0/bpfd-operator/apis/v1alpha1#XdpProgram) via the +[bpfd-operator APIs](https://pkg.go.dev/github.com/bpfd-dev/bpfd@v0.3.0/bpfd-operator/apis/v1alpha1). More info about bpfd can be found [here](https://bpfd.dev/#challenges-for-ebpf-in-kubernetes). +Assuming a bpfd (Kind) cluster is deployed, AF_XDP DP can be deployed in that cluster by issuing the following commands: + +```bash +#!/bin/bash +make docker +docker pull quay.io/mtahhan/cndp-map-pinning:latest +docker pull quay.io/mtahhan/xsk_def_xdp_prog:latest +make setup-multus +make KIND_CLUSTER_NAME=bpfd-deployment kind-label-bpfd-cp +make KIND_CLUSTER_NAME=bpfd-deployment IMAGE=quay.io/mtahhan/cndp-map-pinning kind-load-custom-image +make KIND_CLUSTER_NAME=bpfd-deployment IMAGE=quay.io/mtahhan/xsk_def_xdp_prog kind-load-custom-image +make KIND_CLUSTER_NAME=bpfd-deployment kind-deploy-bpfd +``` + #### AF_XDP Bytecode image bpfd works with eBPF Bytecode packaged in an OCI compliant image. More info on this can be found @@ -561,30 +578,31 @@ that holds the AF_XDP bpf program bytecode. An example of such a program exists https://github.com/bpfd-dev/bpfd.git ``` -And build docker image that specifies the `PROGRAM_TYPE`, `BYTECODE_FILENAME`, `SECTION_NAME` and `KERNEL_COMPILE_VER`. +And build docker image that specifies the `PROGRAM_TYPE`, `BPF_FUNCTION_NAME`, `PROGRAM_NAME` ,`BYTECODE_FILENAME`, `SECTION_NAME` and `KERNEL_COMPILE_VER`. For example for the default AF_XDP eBPF program from xdp-tools [xsk_def_xdp_prog.c](https://github.com/xdp-project/xdp-tools/blob/master/lib/libxdp/xsk_def_xdp_prog.c) ```bash -docker build --build-arg PROGRAM_NAME=xsk_def_xdp_prog --build-arg PROGRAM_TYPE=xdp --build-arg BYTECODE_FILENAME=xsk_def_xdp_prog.o --build-arg SECTION_NAME=xsk_def_prog --build-arg KERNEL_COMPILE_VER=$(uname -r) -f packaging/container-deployment/Containerfile.bytecode /$PATH_TO_XDP_TOOLS/xdp-tools/lib/libxdp -t quay.io/$USER/xsk_def_xdp_prog:latest +docker build --build-arg PROGRAM_NAME=xsk_def_xdp_prog --build-arg BPF_FUNCTION_NAME=xsk_def_prog --build-arg PROGRAM_TYPE=xdp --build-arg BYTECODE_FILENAME=xsk_def_xdp_prog.o --build-arg KERNEL_COMPILE_VER=$(uname -r) -f packaging/container-deployment/Containerfile.bytecode /$PATH_TO_XDP_TOOLS/xdp-tools/lib/libxdp -t quay.io/$USER/xsk_def_xdp_prog:latest ``` -Two pool configuration parameters have been added to allow the specification of the default OCI bytecode image and section to use, +Two pool configuration parameters have been added to allow the specification of the OCI bytecode image and section to use, an example is shown below: ```yaml - "bpfByteCodeImage": "quay.io/bpfd-bytecode/go-xdp-counter", - "bpfByteCodeFunction": "xdp_stats", + "BpfByteCodeImage": "quay.io/mtahhan/xsk_def_xdp_prog:latest", + "BpfByteCodeFunction": "xsk_def_prog", ``` -In the future, we would like to make the eBPF program/bytecode selection configurable via pod annotation. - -#### DPCNIServer +#### DPCNISyncerServer The bpfd integration and map pinning support in the DP introduced the need for a syncronization call from the CNI to the DP in order to delete/unload the XDP program and any resources for an interface being returned to the host on pod deletion. Originally the BPF program unload functionality was part of the CNI itself. + **Note:** This is enabled on the CNI through +the `dpSyncer` CNI configuration parameter . An example configuration is shown in [nad_with_syncer.yaml](examples/nad_with_syncer.yaml) + ## CLOC Output from CLOC (count lines of code) - github.com/AlDanial/cloc diff --git a/examples/nad_with_syncer.yaml b/examples/nad_with_syncer.yaml index 75bed207..997da707 100644 --- a/examples/nad_with_syncer.yaml +++ b/examples/nad_with_syncer.yaml @@ -1,21 +1,19 @@ # WARNING: This is an example definition only. Remove all comments before use. - apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: - name: afxdp-network + name: afxdp-network # Name of this network, pods will request this network by name annotations: - k8s.v1.cni.cncf.io/resourceName: afxdp/myPool + k8s.v1.cni.cncf.io/resourceName: afxdp/myPool # Needs to match the device plugin pool name / resource type spec: config: '{ "cniVersion": "0.3.0", - "type": "afxdp", - "mode": "primary", - "logFile": "afxdp-cni.log", - "logLevel": "debug", - "dpSyncer": true, - "skipUnloadBpf": true, - "ipam": { + "type": "afxdp", # CNI binary, leave as afxdp + "mode": "primary", # CNI mode setting (required) + "logFile": "afxdp-cni.log", # CNI log file location (optional) + "logLevel": "debug", # CNI logging level (optional) + "dpSyncer": true, # Sync with Device Plugin over gRPC MUST BE SET for bpf map pinning + "ipam": { # CNI IPAM plugin and associated config (optional) "type": "host-local", "subnet": "192.168.1.0/24", "rangeStart": "192.168.1.200", diff --git a/go.mod b/go.mod index fbd72762..04340a65 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/bpfd-dev/bpfd v0.3.0 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.1.1 - github.com/distribution/reference v0.5.0 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/golang/protobuf v1.5.3 github.com/google/gofuzz v1.2.0 diff --git a/go.sum b/go.sum index bd7e8390..79bdac34 100644 --- a/go.sum +++ b/go.sum @@ -731,8 +731,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.65.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3/go.mod h1:gt38b7cvVKazi5XkHvINNytZXgTEntyhtyM3HQz46Nk= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -1392,7 +1390,6 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= diff --git a/internal/bpf/xdp-pass/xdp_pass.c b/internal/bpf/xdp-pass/xdp_pass.c index b33becab..a5fa6fbb 100644 --- a/internal/bpf/xdp-pass/xdp_pass.c +++ b/internal/bpf/xdp-pass/xdp_pass.c @@ -17,7 +17,7 @@ #include #include // clang-format on -SEC("xdp/pass") +SEC("xdp") int xdp_pass(struct xdp_md *ctx) { return XDP_PASS; } char _license[] SEC("license") = "Dual BSD";