Add Support for TCX Programs

This PR introduces support for TCX based on the current implementation
in the astoycos Aya TCX branch. While minor API changes are
anticipated before Aya TCX merges, these are expected to be cosmetic
and should not significantly affect bpfman.

The PR implements a priority-based API, similar to the existing TC
API, with a key difference: TCX does not support "proceed_on"
configuration as it's not available in the kernel API. Flow
continuation is determined by the return value of the current program.

It's important to note that neither the Aya nor the kernel APIs
directly support priority. They utilize a relative positioning system,
requiring new TCX programs to be placed relative to existing ones. To
maintain usability across multiple independent applications, we’ve
preserved a TC-like priority API in bpfman, which internally maps to
Aya’s relative-ordering API.

This PR also adds an integration test and example programs for TCX,
which have been successfully tested with Mohammed’s TCX bpfman-operator
PR (bpfman/bpfman-operator#102).

Additionally, due to TCX's requirement for kernel version 6.6 or
higher, GitHub Actions have been updated to use ubuntu-24.04.

Signed-off-by: Andre Fredette <[email protected]>

.github/workflows/build.yml

@@ -15,7 +15,7 @@ env:
 
 jobs:
   check-license:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 3
 
     steps:
@@ -44,7 +44,7 @@ jobs:
             # rust-target: s390x-unknown-linux-gnu
             # filename: linux-s390x
             # command: cross
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Install dependencies
         # Dependencies only needed to build eBPF and used by integration-tests, only on amd64
@@ -280,7 +280,7 @@ jobs:
             filename: linux-ppc64le
           - arch: s390x
             filename: linux-s390x
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout bpfman
         uses: actions/checkout@v4
@@ -315,7 +315,7 @@ jobs:
           GOARCH=${{ matrix.arch.arch }} make build
 
   basic-integration-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: [build, build-go]
     steps:
       - name: Set up environment for running integration tests from manual trigger
@@ -330,6 +330,7 @@ jobs:
           echo "KRETPROBE_IMAGE_LOC=quay.io/bpfman-bytecode/kretprobe:${GITHUB_REF_NAME}" >> $GITHUB_ENV
           echo "XDP_COUNTER_IMAGE_LOC=quay.io/bpfman-bytecode/go-xdp-counter:${GITHUB_REF_NAME}" >> $GITHUB_ENV
           echo "TC_COUNTER_IMAGE_LOC=quay.io/bpfman-bytecode/go-tc-counter:${GITHUB_REF_NAME}" >> $GITHUB_ENV
+          echo "TCX_COUNTER_IMAGE_LOC=quay.io/bpfman-bytecode/go-tcx-counter:${GITHUB_REF_NAME}" >> $GITHUB_ENV
           echo "TRACEPOINT_COUNTER_IMAGE_LOC=quay.io/bpfman-bytecode/go-tracepoint-counter:${GITHUB_REF_NAME}" >> $GITHUB_ENV
           echo "FENTRY_IMAGE_LOC=quay.io/bpfman-bytecode/fentry:${GITHUB_REF_NAME}" >> $GITHUB_ENV
           echo "FEXIT_IMAGE_LOC=quay.io/bpfman-bytecode/fexit:${GITHUB_REF_NAME}" >> $GITHUB_ENV
@@ -413,7 +414,7 @@ jobs:
         run: cargo xtask integration-test
 
   build-docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout bpfman
         uses: actions/checkout@v4
@@ -431,7 +432,7 @@ jobs:
 
   coverage:
     needs: [build, build-go]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Download rust coverage artifacts
         uses: actions/download-artifact@v4
@@ -451,7 +452,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/v')
     needs: [build]
     environment: crates.io
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout bpfman
         uses: actions/checkout@v4
@@ -501,7 +502,7 @@ jobs:
         coverage,
         basic-integration-tests,
       ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Build Complete
         run: echo "Build Complete"

.github/workflows/docs-build.yaml

@@ -8,7 +8,7 @@ on: # yamllint disable-line rule:truthy
 
 jobs:
   build-docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 3
     steps:
       - uses: actions/checkout@v4

.github/workflows/image-build.yaml

@@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout bpfman
         uses: actions/checkout@v4
@@ -37,7 +37,7 @@ jobs:
       packages: write
       id-token: write # needed for signing the images with GitHub OIDC Token
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -84,6 +84,20 @@ jobs:
               # set latest tag for default branch
               type=raw,value=latest,enable={{is_default_branch}}
 
+          - registry: quay.io
+            # build_language: go - Not building locally, so don't install go tools
+            repository: bpfman-userspace
+            image: go-tcx-counter
+            context: .
+            dockerfile: ./examples/go-tcx-counter/container-deployment/Containerfile.go-tcx-counter
+            tags: |
+              type=ref,event=branch
+              type=ref,event=tag
+              type=ref,event=pr
+              type=sha,format=long
+              # set latest tag for default branch
+              type=raw,value=latest,enable={{is_default_branch}}
+
           - registry: quay.io
             # build_language: go - Not building locally, so don't install go tools
             repository: bpfman-userspace
@@ -233,7 +247,7 @@ jobs:
       packages: write
       id-token: write # needed for signing the images with GitHub OIDC Token
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -270,6 +284,22 @@ jobs:
               # set latest tag for default branch
               type=raw,value=latest,enable={{is_default_branch}}
 
+          - registry: quay.io
+            build_language: go
+            bpf_build_wrapper: go
+            repository: bpfman-bytecode
+            image: go-tcx-counter
+            context: .
+            dockerfile: ./Containerfile.bytecode.multi.arch
+            bytecode_dir: ./examples/go-tcx-counter
+            tags: |
+              type=ref,event=branch
+              type=ref,event=tag
+              type=ref,event=pr
+              type=sha,format=long
+              # set latest tag for default branch
+              type=raw,value=latest,enable={{is_default_branch}}
+
           - registry: quay.io
             build_language: go
             bpf_build_wrapper: go
@@ -398,6 +428,22 @@ jobs:
               # set latest tag for default branch
               type=raw,value=latest,enable={{is_default_branch}}
 
+          - registry: quay.io
+            build_language: rust
+            bpf_build_wrapper: rust
+            repository: bpfman-bytecode
+            image: tcx_test
+            context: .
+            dockerfile: ./Containerfile.bytecode.multi.arch
+            bytecode_dir: ./tests/integration-test/bpf/.output/tcx_test.bpf
+            tags: |
+              type=ref,event=branch
+              type=ref,event=tag
+              type=ref,event=pr
+              type=sha,format=long
+              # set latest tag for default branch
+              type=raw,value=latest,enable={{is_default_branch}}
+
           - registry: quay.io
             build_language: rust
             bpf_build_wrapper: rust

Cargo.toml

@@ -62,6 +62,7 @@ opentelemetry-otlp = { version = "0.15.0", default-features = false }
 opentelemetry-semantic-conventions = { version = "0.14.0" }
 opentelemetry_sdk = { version = "0.22.1", default-features = false }
 predicates = { version = "3.1.2", default-features = false }
+procfs = { version = "0.16.0", default-features = false }
 prost = { version = "0.12.6", default-features = false }
 prost-types = { version = "0.12.6", default-features = false }
 public-api = { version = "0.35.0", default-features = false }

bpfman-api/src/bin/rpc/rpc.rs

@@ -4,15 +4,16 @@ use bpfman::{
     add_program, get_program, list_programs, pull_bytecode, remove_program,
     types::{
         FentryProgram, FexitProgram, KprobeProgram, ListFilter, Location, Program, ProgramData,
-        TcProceedOn, TcProgram, TracepointProgram, UprobeProgram, XdpProceedOn, XdpProgram,
+        TcProceedOn, TcProgram, TcxProgram, TracepointProgram, UprobeProgram, XdpProceedOn,
+        XdpProgram,
     },
 };
 use bpfman_api::v1::{
     attach_info::Info, bpfman_server::Bpfman, bytecode_location::Location as RpcLocation,
     list_response::ListResult, FentryAttachInfo, FexitAttachInfo, GetRequest, GetResponse,
     KprobeAttachInfo, ListRequest, ListResponse, LoadRequest, LoadResponse, PullBytecodeRequest,
-    PullBytecodeResponse, TcAttachInfo, TracepointAttachInfo, UnloadRequest, UnloadResponse,
-    UprobeAttachInfo, XdpAttachInfo,
+    PullBytecodeResponse, TcAttachInfo, TcxAttachInfo, TracepointAttachInfo, UnloadRequest,
+    UnloadResponse, UprobeAttachInfo, XdpAttachInfo,
 };
 use tonic::{Request, Response, Status};
 
@@ -91,6 +92,21 @@ impl Bpfman for BpfmanLoader {
                     .map_err(|e| Status::aborted(format!("failed to create tcprogram: {e}")))?,
                 )
             }
+            Info::TcxAttachInfo(TcxAttachInfo {
+                priority,
+                iface,
+                position: _,
+                direction,
+            }) => {
+                let direction = direction
+                    .try_into()
+                    .map_err(|_| Status::aborted("direction is not a string"))?;
+                Program::Tcx(
+                    TcxProgram::new(data, priority, iface, direction).map_err(|e| {
+                        Status::aborted(format!("failed to create tcxprogram: {e}"))
+                    })?,
+                )
+            }
             Info::TracepointAttachInfo(TracepointAttachInfo { tracepoint }) => Program::Tracepoint(
                 TracepointProgram::new(data, tracepoint)
                     .map_err(|e| Status::aborted(format!("failed to create tcprogram: {e}")))?,

bpfman-api/src/bpfman.v1.rs

@@ -1,3 +1,4 @@
+// This file is @generated by prost-build.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct BytecodeImage {
@@ -117,6 +118,18 @@ pub struct TcAttachInfo {
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TcxAttachInfo {
+    #[prost(int32, tag = "1")]
+    pub priority: i32,
+    #[prost(string, tag = "2")]
+    pub iface: ::prost::alloc::string::String,
+    #[prost(int32, tag = "3")]
+    pub position: i32,
+    #[prost(string, tag = "4")]
+    pub direction: ::prost::alloc::string::String,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct TracepointAttachInfo {
     #[prost(string, tag = "1")]
     pub tracepoint: ::prost::alloc::string::String,
@@ -164,7 +177,7 @@ pub struct FexitAttachInfo {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct AttachInfo {
-    #[prost(oneof = "attach_info::Info", tags = "2, 3, 4, 5, 6, 7, 8")]
+    #[prost(oneof = "attach_info::Info", tags = "2, 3, 4, 5, 6, 7, 8, 9")]
     pub info: ::core::option::Option<attach_info::Info>,
 }
 /// Nested message and enum types in `AttachInfo`.
@@ -186,6 +199,8 @@ pub mod attach_info {
         FentryAttachInfo(super::FentryAttachInfo),
         #[prost(message, tag = "8")]
         FexitAttachInfo(super::FexitAttachInfo),
+        #[prost(message, tag = "9")]
+        TcxAttachInfo(super::TcxAttachInfo),
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]

bpfman-api/src/lib.rs

@@ -10,8 +10,8 @@ use crate::v1::{
     attach_info::Info, bytecode_location::Location as V1Location, AttachInfo,
     BytecodeImage as V1BytecodeImage, BytecodeLocation, FentryAttachInfo, FexitAttachInfo,
     KernelProgramInfo as V1KernelProgramInfo, KprobeAttachInfo, ProgramInfo,
-    ProgramInfo as V1ProgramInfo, TcAttachInfo, TracepointAttachInfo, UprobeAttachInfo,
-    XdpAttachInfo,
+    ProgramInfo as V1ProgramInfo, TcAttachInfo, TcxAttachInfo, TracepointAttachInfo,
+    UprobeAttachInfo, XdpAttachInfo,
 };
 
 #[path = "bpfman.v1.rs"]
@@ -57,6 +57,12 @@ impl TryFrom<&Program> for ProgramInfo {
                     direction: p.get_direction()?.to_string(),
                     proceed_on: p.get_proceed_on()?.as_action_vec(),
                 })),
+                Program::Tcx(p) => Some(Info::TcxAttachInfo(TcxAttachInfo {
+                    priority: p.get_priority()?,
+                    iface: p.get_iface()?.to_string(),
+                    position: p.get_current_position()?.unwrap_or(0) as i32,
+                    direction: p.get_direction()?.to_string(),
+                })),
                 Program::Tracepoint(p) => Some(Info::TracepointAttachInfo(TracepointAttachInfo {
                     tracepoint: p.get_tracepoint()?.to_string(),
                 })),

bpfman/src/bin/cli/args.rs

@@ -185,6 +185,26 @@ pub(crate) enum LoadCommands {
         proceed_on: Vec<String>,
     },
     #[command(disable_version_flag = true)]
+    /// Install an eBPF program on the TCX hook point for a given interface and
+    /// direction.
+    Tcx {
+        /// Required: Direction to apply program.
+        ///
+        /// [possible values: ingress, egress]
+        #[clap(short, long, verbatim_doc_comment)]
+        direction: String,
+
+        /// Required: Interface to load program on.
+        #[clap(short, long)]
+        iface: String,
+
+        /// Optional: Priority to run program in chain. Lower value runs first.
+        /// [possible values: 1-1000]
+        /// [default: 1000]
+        #[clap(short, long)]
+        priority: i32,
+    },
+    #[command(disable_version_flag = true)]
     /// Install an eBPF program on a Tracepoint.
     Tracepoint {
         /// Required: The tracepoint to attach to.

bpfman/src/bin/cli/load.rs

@@ -8,7 +8,7 @@ use bpfman::{
     add_program,
     types::{
         FentryProgram, FexitProgram, KprobeProgram, Location, Program, ProgramData, TcProceedOn,
-        TcProgram, TracepointProgram, UprobeProgram, XdpProceedOn, XdpProgram,
+        TcProgram, TcxProgram, TracepointProgram, UprobeProgram, XdpProceedOn, XdpProgram,
     },
 };
 
@@ -113,6 +113,22 @@ impl LoadCommands {
                     direction.to_string().try_into()?,
                 )?))
             }
+            LoadCommands::Tcx {
+                direction,
+                iface,
+                priority,
+            } => {
+                match direction.as_str() {
+                    "ingress" | "egress" => (),
+                    other => bail!("{} is not a valid direction", other),
+                };
+                Ok(Program::Tcx(TcxProgram::new(
+                    data,
+                    *priority,
+                    iface.to_string(),
+                    direction.to_string().try_into()?,
+                )?))
+            }
             LoadCommands::Tracepoint { tracepoint } => Ok(Program::Tracepoint(
                 TracepointProgram::new(data, tracepoint.to_string())?,
             )),