Add cilium as filter for benchmark

README.md

@@ -22,36 +22,68 @@ This repository contains a set of tools to measure the egress filtering performa
 
     ```bash
     git clone [email protected]:kinvolk/egress-filtering-benchmark.git
-    cd lokomotive
+    cd lokomotive/calico-benchmark-cluster
     lokoctl cluster apply
     ```
 
-     Set location of kubeconfig using the environment variable **KUBECONFIG**:
+     Set location of kubeconfig in the environment variable **KUBECONFIG_CALICO**:
 
      ```
      cd egress-filtering-benchmark/lokomotive
-     export KUBECONFIG=$PWD/assets/cluster-assets/auth/kubeconfig
+     export KUBECONFIG_CALICO=$PWD/assets/cluster-assets/auth/kubeconfig
      ```
 
      Label the worker node as follows:
      ```
      kubectl label node calico-benchmark-pool-1-worker-0 nodetype=worker-benchmark
      ```
+4. Create a Kubernetes cluster with Cilium using [Lokomotive](https://github.com/kinvolk/lokomotive) with at least one worker node.
+
+    A minimal working configuration that can be deployed on [Packet](https://www.packet.com/) (acquired by [Equinix Metal](https://metal.equinix.com/))
+
+    Since Lokomotive doesn't ship with Cilium in the lokoctl binary, we will checkout the project and
+    build the binary based on a different branch.
+    ```bash
+    git clone [email protected]:kinvolk/lokomotive.git
+    cd lokomotive
+    git checkout imran/cilium-instead-of-calico
+    make
+    mv lokoctl ~/.local/bin/lokoctl_cilium
+    ```
+    Update the variables in `lokocfg.vars` and execute:
+
+    ```bash
+    git clone [email protected]:kinvolk/egress-filtering-benchmark.git
+    cd lokomotive/cilium-benchmark-cluster
+    lokoctl_cilium cluster apply
+    ```
+
+     Set location of kubeconfig using the environment variable **KUBECONFIG_CILIUM**:
+
+     ```
+     cd egress-filtering-benchmark/lokomotive
+     export KUBECONFIG_CILIUM=$PWD/assets/cluster-assets/auth/kubeconfig
+     ```
+
+     Label the worker node as follows:
+     ```
+     kubectl label node cilium-benchmark-pool-1-worker-0 nodetype=worker-benchmark
+     ```
 
-4. Configure the parameters of the test in the [parameters.py](benchmark/parameters.py) file.
+5. Configure the parameters of the test in the [parameters.py](benchmark/parameters.py) file.
 
-5. Install the required libraries in the client to run the Python script
+6. Install the required libraries in the client to run the Python script
 
 ```
 pip install -r requirements.txt
 ```
 
-6. Execute the tests:
+7. Execute the tests:
 
 ```
 $ cd benchmark
 $ make
-$ python run_tests.py --mode udp --username USERNAME --client CLIENTADDR --server SERVERADDR
+$ python run_tests.py --mode udp --username USERNAME --client CLIENTADDR --server SERVERADDR --kubeconfig-cilium-cluster $KUBECONFIG_CILIUM --kubeconfig-calico-cluster $KUBECONFIG_CALICO
 ```
 
 This will create some csv files with the information about the test.
@@ -65,6 +97,13 @@ $ python plot_data.py
 
 This will create some svg files with the graphs.
 
+8. Cleanup the Lokomotive clusters created in the benchmark process:
+  ```bash
+  cd lokomotive/calico-benchmark-cluster
+  lokoctl cluster destroy --confirm
+  cd lokomotive/cilium-benchmark-cluster
+  lokoctl_cilium cluster destroy --confirm
+  ```
 ## Credits
 
 The BPF filter is inspired by the [tc-bpf](https://man7.org/linux/man-pages/man8/tc-bpf.8.html) man page and the [Cilium documentation](https://docs.cilium.io/en/latest/bpf/#tc-traffic-control).

benchmark/k8s.py

@@ -45,6 +45,30 @@
   name: benchmark-sa
   namespace: default
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  # "namespace" omitted since ClusterRoles are not namespaced
+  name: benchmark-cilium-resource-creator
+rules:
+- apiGroups: ["cilium.io"]
+  #resources: ["*"]
+  resources: ["ciliumclusterwidenetworkpolicies"]
+  verbs: ["get", "watch", "list", "create", "delete","update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: benchmark-cilium-resource-creator
+roleRef:
+  kind: ClusterRole
+  name: benchmark-cilium-resource-creator
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: benchmark-sa
+  namespace: default
+---
 apiVersion: v1
 kind: Pod
 metadata:
@@ -58,7 +82,7 @@
     nodetype: worker-benchmark
   containers:
   - name: benchmark
-    image: quay.io/imran/benchmark:v0.20
+    image: quay.io/imran/benchmark:v0.21
     imagePullPolicy: Always
     securityContext:
       privileged: true

benchmark/main.go

@@ -1,27 +1,21 @@
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"time"
 
-	"net"
-
-	tcbpf "github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/tc-bpf"
-	cgroupbpf "github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/cgroup-bpf"
 	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/calico"
+	cgroupbpf "github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/cgroup-bpf"
+	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/cilium"
 	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/ipset"
 	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/iptables"
+	tcbpf "github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/tc-bpf"
+	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/filters/util"
 	"github.com/kinvolk/k8s-egress-filtering-benchmark/pkg/ipnetsgenerator"
-
-	"github.com/sparrc/go-ping"
-)
-
-const (
-	testIPDefault = "8.8.8.8" // IP used to test if the filter works.
 )
 
 var (
@@ -35,7 +29,7 @@ var (
 
 type filter interface {
 	SetUp(nets []net.IPNet, iface string) (int64, error)
-	CleanUp()
+	CleanUp() error
 }
 
 func init() {
@@ -44,7 +38,7 @@ func init() {
 	flag.StringVar(&ipnetsParam, "ipnets", "", "List of ipnets and their weigth to generate (ex. 24:0.7,16:0.1)")
 	flag.Int64Var(&seed, "seed", 0, "Seed to use for the random generator")
 	flag.StringVar(&filterType, "filter", "", "Type of filter to use (tc-bpf, cgroup-bpf, iptables, ipset)")
-	flag.StringVar(&testIP, "test-ip", testIPDefault, "IP to perform a ping to test if filters were correctly applied")
+	flag.StringVar(&testIP, "test-ip", util.DefaultPingIP, "IP to perform a ping to test if filters were correctly applied")
 }
 
 func main() {
@@ -79,6 +73,8 @@ func main() {
 		filter = ipset.New()
 	case "calico":
 		filter = calico.New(nets, iface)
+	case "cilium":
+		filter = cilium.New(nets, iface, testIP)
 	default:
 		fmt.Printf("%q is not a valid filter type", filterType)
 		os.Exit(1)
@@ -87,14 +83,18 @@ func main() {
 	var setupTime int64
 
 	if filter != nil {
-		// Check that the test ip is reachable before applying the filter
-		if err := checkPingSuccess(testIP); err != nil {
+		// Check that the test ip is reachable before applying the filter.
+		if err := util.CheckPingSuccess(testIP); err != nil {
 			fmt.Printf("%s\n", err)
 			return
 		}
-		if err := checkDNSSuccess(testIP); err != nil {
-			fmt.Printf("%s\n", err)
-			return
+		// Disabling this check for Calico, as Calico has fail safe network policies
+		// that allow DNS communication.
+		if filterType != "calico" {
+			if err := util.CheckDNSSuccess(testIP); err != nil {
+				fmt.Printf("%s\n", err)
+				return
+			}
 		}
 
 		var err error
@@ -103,18 +103,26 @@ func main() {
 			fmt.Printf("error setting up filter %s", err)
 			return
 		}
-
-		// Check that the test ip is not reachable after applying the filter
-		if err := checkPingFail(testIP); err != nil {
+		// Check that the test ip is not reachable after applying the filter.
+		if err := util.CheckPingFail(testIP); err != nil {
 			fmt.Printf("%s\n", err)
 			return
 		}
-		if err := checkDNSFail(testIP); err != nil {
-			fmt.Printf("%s\n", err)
-			return
+		// Disabling this check for Calico, as Calico has fail safe network policies
+		// that allow DNS communication.
+		if filterType != "calico" {
+			if err := util.CheckDNSFail(testIP); err != nil {
+				fmt.Printf("%s\n", err)
+				return
+			}
 		}
 
-		defer filter.CleanUp()
+		defer func() {
+			if ferr := filter.CleanUp(); ferr != nil {
+				fmt.Printf("%v\n", ferr)
+				return
+			}
+		}()
 	}
 
 	if os.Getenv("BENCHMARK_COMMAND") == "MEASURE_SETUP_TIME" {
@@ -133,74 +141,3 @@ func main() {
 		fmt.Scanf("%s", &input)
 	}
 }
-
-func createPinger() (*ping.Pinger, error) {
-	pinger, err := ping.NewPinger(testIP)
-	if err != nil {
-		return nil, err
-	}
-
-	pinger.Count = 1
-	pinger.SetPrivileged(true)
-	pinger.Timeout = time.Second
-	return pinger, err
-}
-
-func checkPingSuccess(ip string) error {
-	pinger, err := createPinger()
-	if err != nil {
-		return err
-	}
-
-	pinger.Run()
-	if pinger.PacketsRecv != 1 {
-		return fmt.Errorf("imposible to ping %q", ip)
-	}
-
-	return nil
-}
-
-func checkPingFail(ip string) error {
-	pinger, err := createPinger()
-	if err != nil {
-		return err
-	}
-
-	pinger.Run()
-	if pinger.PacketsRecv != 0 {
-		return fmt.Errorf("ping to %q should have fail", ip)
-	}
-
-	return nil
-}
-
-func checkDNSSuccess(ip string) error {
-	r := &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Second * time.Duration(5),
-			}
-			return d.DialContext(ctx, "udp", ip+":53")
-		},
-	}
-	_, err := r.LookupHost(context.Background(), "www.google.com")
-	return err
-}
-
-func checkDNSFail(ip string) error {
-	r := &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Second * time.Duration(5),
-			}
-			return d.DialContext(ctx, "udp", ip+":53")
-		},
-	}
-	_, err := r.LookupHost(context.Background(), "www.google.com")
-	if err == nil {
-	   return fmt.Errorf("DNS request to %s:53 should have failed", ip)
-	}
-	return nil
-}

benchmark/parameters.py

@@ -1,7 +1,7 @@
 # change these values to setup the test
 
 # List of filters to use. Supported values are none, bpf, iptables and ipset.
-filters = ['none','tc-bpf','cgroup-bpf','iptables','ipset','calico']
+filters = ['none','tc-bpf','cgroup-bpf','iptables','ipset','calico','cilium']
 # Number of rules to test
 number_rules = ['10','100','1000','10000','100000','1000000']
 # Number of iterations to run each test.

benchmark/plot_data.py

@@ -47,7 +47,7 @@ def get_plotable_tables(data):
             err[filter_index][rules_index] = t_err = np.std(values, ddof=1)/np.sqrt(len(values))
     return (avg, err)
 
-bar_color = ("#09BAC8", "#FFF200", "#F72E5C", "#17B838")
+bar_color = ("#09BAC8", "#FFF200", "#F72E5C", "#17B838","#F6830F","#892CDC")
 ecolor = ""
 grid_color = "#B0B0B0"
 axis_heading = "#585858"