Skip to content
This repository has been archived by the owner on Mar 20, 2024. It is now read-only.

Commit

Permalink
multiple device request issue (#81)
Browse files Browse the repository at this point in the history 
* multiple device request issue

the issue is if a single pod requests different devices from
different pools it results in multiple uds servers serving the
same container and all attempt to mount their uds to the pod
as /tmp/afxdp.sock.
A similar issue exists for the AFXDP_DEVICES env var that's
set in each container. This patch fixes the first issue by
mounting the xsksocket at /tmp/afxdp_dp/<netdev>/afxdp.sock,
a similar issue also existed for the bpf map pinning support.

---------

Signed-off-by: Maryam Tahhan <[email protected]>
  • Loading branch information
@maryamtahhan
maryamtahhan authored Nov 28, 2023
1 parent 38317c2 commit 720d92b
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 44 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,14 @@ build: builddp buildcni
docker: ## Build docker image
@echo "****** Docker Image ******"
@echo
docker build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
docker build -t afxdp-device-plugin -f images/amd64.dockerfile .
@echo
@echo

podman: ## Build podman image
@echo "****** Podman Image ******"
@echo
podman build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
podman build -t afxdp-device-plugin -f images/amd64.dockerfile .
@echo
@echo

Expand Down
23 changes: 13 additions & 10 deletions constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var (

/* Devices */
devicesProhibited = []string{"eno", "eth", "lo", "docker", "flannel", "cni"} // interfaces we never add to a pool
devicesEnvVar = "AFXDP_DEVICES" // env var set in the end user application pod, lists AF_XDP devices attached
devicesEnvVarPrefix = "AFXDP_DEVICES_" // env var set in the end user application pod, lists AF_XDP devices attached
deviceValidNameRegex = `^[a-zA-Z0-9_-]+$` // regex to check if a string is a valid device name
deviceValidNameMin = 1 // minimum length of a device name
deviceValidNameMax = 50 // maximum length of a device name
Expand Down Expand Up @@ -74,16 +74,17 @@ var (
afxdpMinimumLinux = "4.18.0" // minimum Linux version for AF_XDP support

/* UDS*/
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
udsMsgBufSize = 64 // uds message buffer size
udsCtlBufSize = 4 // uds control buffer size
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
udsPodPath = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
udsMsgBufSize = 64 // uds message buffer size
udsCtlBufSize = 4 // uds control buffer size
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
udsPodPath = "/tmp/afxdp_dp/" // the uds filepath as it will appear in the end user application pod
udsPodSock = "/afxdp.sock"

/* BPF*/
bpfMapPodPath = "/tmp/xsks_map"
bpfMapPodPath = "/tmp/afxdp_dp/"
xsk_map = "/xsks_map"

udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
Expand Down Expand Up @@ -216,6 +217,7 @@ type uds struct {
SockDir string
DirFileMode int
PodPath string
SockName string
Handshake handshake
}

Expand Down Expand Up @@ -284,7 +286,7 @@ func init() {

Devices = devices{
Prohibited: devicesProhibited,
EnvVarList: devicesEnvVar,
EnvVarList: devicesEnvVarPrefix,
ValidNameRegex: deviceValidNameRegex,
ValidNameMin: deviceValidNameMin,
ValidNameMax: deviceValidNameMax,
Expand Down Expand Up @@ -326,6 +328,7 @@ func init() {
SockDir: udsSockDir,
DirFileMode: udsDirFileMode,
PodPath: udsPodPath,
SockName: udsPodSock,
Handshake: handshake{
Version: handshakeHandshakeVersion,
RequestVersion: handshakeRequestVersion,
Expand Down
27 changes: 15 additions & 12 deletions internal/deviceplugin/poolManager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,22 @@ func (pm *PoolManager) Allocate(ctx context.Context,
cresp := new(pluginapi.ContainerAllocateResponse)
envs := make(map[string]string)

if !pm.UdsServerDisable {
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: udsPath,
ContainerPath: constants.Uds.PodPath,
ReadOnly: false,
})
}

//loop each device request per container
for _, devName := range crqt.DevicesIDs {
device := pm.Devices[devName]
pretty, _ := tools.PrettyString(device.Public())
logging.Debugf("Device: %s", pretty)

containerSockPath := constants.Uds.PodPath + device.Name() + constants.Uds.SockName

if !pm.UdsServerDisable {
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: udsPath,
ContainerPath: containerSockPath,
ReadOnly: false,
})
}

if device.Mode() != pm.Mode {
err := fmt.Errorf("pool mode %s does not match device mode %s", pm.Mode, device.Mode())
logging.Errorf("%v", err)
Expand Down Expand Up @@ -265,16 +267,18 @@ func (pm *PoolManager) Allocate(ctx context.Context,

//FULL PATH WILL INCLUDE THE XSKMAP...
fullPath := pinPath + constants.Bpf.Xsk_map
logging.Debugf("mapping %s to %s", fullPath, constants.Bpf.BpfMapPodPath)
containerMapPath := constants.Bpf.BpfMapPodPath + device.Name() + constants.Bpf.Xsk_map
logging.Debugf("mapping %s to %s", fullPath, containerMapPath)
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: fullPath,
ContainerPath: constants.Bpf.BpfMapPodPath,
ContainerPath: containerMapPath,
ReadOnly: false,
})
}
}

envs[constants.Devices.EnvVarList] = strings.Join(crqt.DevicesIDs, " ")
envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)
envs[envVar] = strings.Join(crqt.DevicesIDs, " ")
envsPrint, err := tools.PrettyString(envs)
if err != nil {
logging.Errorf("Error printing container environment variables: %v", err)
Expand All @@ -283,7 +287,6 @@ func (pm *PoolManager) Allocate(ctx context.Context,
}
cresp.Envs = envs
response.ContainerResponses = append(response.ContainerResponses, cresp)

}

if !pm.UdsServerDisable {
Expand Down
67 changes: 47 additions & 20 deletions internal/deviceplugin/poolManager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package deviceplugin
import (
"context"
"encoding/json"
"strings"
"testing"

"github.com/intel/afxdp-plugins-for-kubernetes/constants"
Expand Down Expand Up @@ -57,6 +58,8 @@ func TestAllocate(t *testing.T) {
pm.ServerFactory = udsserver.NewFakeServerFactory()
pm.BpfHandler = bpf.NewFakeHandler()

envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)

testCases := []struct {
name string
containerRequests []*pluginapi.ContainerAllocateRequest
Expand All @@ -69,10 +72,10 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
Envs: map[string]string{envVar: "dev_1"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -90,10 +93,20 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -112,10 +125,10 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
Envs: map[string]string{envVar: "dev_1"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -124,10 +137,10 @@ func TestAllocate(t *testing.T) {
Annotations: map[string]string{},
},
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_2"},
Envs: map[string]string{envVar: "dev_2"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -146,10 +159,20 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -158,10 +181,20 @@ func TestAllocate(t *testing.T) {
Annotations: map[string]string{},
},
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_4 dev_5 dev_6"},
Envs: map[string]string{envVar: "dev_4 dev_5 dev_6"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_4" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_5" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_6" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -179,14 +212,8 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: ""},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
},
Envs: map[string]string{envVar: ""},
Mounts: []*pluginapi.Mount{},
Devices: []*pluginapi.DeviceSpec{},
Annotations: map[string]string{},
},
Expand Down

0 comments on commit 720d92b

Please sign in to comment.