create cilium BGPPeeringPolicy automatically

README.md

@@ -34,7 +34,7 @@ For general feature and usage notes, refer to the [Getting Started with Linode N
 
 #### Using IP Sharing instead of NodeBalancers
 Alternatively, the Linode CCM can integrate with [Cilium's BGP Control Plane](https://docs.cilium.io/en/stable/network/bgp-control-plane/)
-to perform load-balancing via IP sharing on Nodes selected by an existing `CiliumBGPPeeringPolicy` in the cluster. This option does not create a backing NodeBalancer and instead
+to perform load-balancing via IP sharing on labelled Nodes. This option does not create a backing NodeBalancer and instead
 provisions a new IP on an ip-holder Nanode to share. See [Shared IP LoadBalancing](#shared-ip-load-balancing).
 
 #### Annotations
@@ -90,22 +90,10 @@ Key | Values | Default | Description
 
 Services of `type: LoadBalancer` can receive an external IP not backed by a NodeBalancer if `--bgp-node-selector` is set on the Linode CCM and either `--default-load-balancer` is set to `cilium-bgp` or the Service has the `service.beta.kubernetes.io/linode-loadbalancer-type` annotation set to `cilium-bgp`. Additionally, the `LINODE_URL` environment variable in the linode CCM needs to be set to "https://api.linode.com/v4beta".
 
-This feature requires the Kubernetes cluster to be using [Cilium](https://cilium.io/) as the CNI with the `bgp-control-plane` feature enabled and a `CiliumBGPPeeringPolicy` applied to the cluster with a node selector specified in the `--bgp-node-selector` flag.
+This feature requires the Kubernetes cluster to be using [Cilium](https://cilium.io/) as the CNI with the `bgp-control-plane` feature enabled.
 
 Example configuration:
 
-```
-apiVersion: "cilium.io/v2alpha1"
-kind: CiliumBGPPeeringPolicy
-metadata:
-  name: 01-bgp-peering-policy
-spec:
-  nodeSelector:
-    matchLabels:
-      cilium-bgp-peering: "true"
-...
-```
-
 ```
 apiVersion: apps/v1
 kind: DaemonSet

cloud/annotations/annotations.go

@@ -33,6 +33,7 @@ const (
 	AnnLinodeHostUUID      = "node.k8s.linode.com/host-uuid"
 
 	// AnnLinodeLoadBalancerType is the annotation used to specify which type of load-balancing solution
-	// to use for the Service. Options are nodebalancer and cilium-bgp. Defaults to nodebalancer.
+	// to use for the Service. Options are nodebalancer and cilium-bgp. Defaults to the default-load-balancer
+	// flag value if this annotation is not set on a Service.
 	AnnLinodeLoadBalancerType = "service.beta.kubernetes.io/linode-loadbalancer-type"
 )

cloud/linode/cilium_loadbalancers.go

@@ -14,24 +14,59 @@ import (
 	"github.com/google/uuid"
 	"github.com/linode/linodego"
 	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 )
 
 const (
-	ciliumLBClass = "io.cilium/bgp-control-plane"
-	ipHolderLabel = "linode-ccm-ip-holder"
+	ciliumLBClass              = "io.cilium/bgp-control-plane"
+	ipHolderLabel              = "linode-ccm-ip-holder"
+	ciliumBGPPeeringPolicyName = "linode-ccm-bgp-peering"
 )
 
-var noBGPSelector = errors.New("no BGP node selector set to configure IP sharing")
+// This mapping is unfortunately necessary since there is no way to get the
+// numeric ID for a data center from the API.
+// These values come from https://www.linode.com/docs/products/compute/compute-instances/guides/failover/#ip-sharing-availability
+var (
+	regionIDMap = map[string]int{
+		"us-southeast": 4,  // Atlanta, GA (USA)
+		"us-ord":       18, // Chicago, IL (USA)
+		"us-central":   2,  // Dallas, TX (USA)
+		// "us-west"   : 3,  // Fremont, CA (USA) UNDERGOING NETWORK UPGRADES
+		"us-lax":     30, // Los Angeles, CA (USA)
+		"us-mia":     28, // Miami, FL (USA)
+		"us-east":    6,  // Newark, NJ (USA)
+		"us-sea":     20, // Seattle, WA (USA)
+		"us-iad":     17, // Washington, DC (USA)
+		"ca-central": 15, // Toronto (Canada)
+		"br-gru":     21, // São Paulo (Brazil)
+		// EMEA
+		"nl-ams":     22, // Amsterdam (Netherlands)
+		"eu-central": 10, // Frankfurt (Germany)
+		"eu-west":    7,  // London (United Kingdom)
+		"it-mil":     27, // Milan (Italy)
+		"ap-west":    14, // Mumbai (India)
+		"fr-par":     19, // Paris (France)
+		"se-sto":     23, // Stockholm (Sweden)
+		// APAC
+		"in-maa":       25, // Chennai (India)
+		"id-cgk":       29, // Jakarta (Indonesia)
+		"jp-osa":       26, // Osaka (Japan)
+		"ap-south":     9,  // Singapore
+		"ap-southeast": 16, // Sydney (Australia)
+		"ap-northeast": 11, // Tokyo (Japan)
+	}
+	errNoBGPSelector = errors.New("no BGP node selector set to configure IP sharing")
+)
 
 // createSharedIP requests an additional IP that can be shared on Nodes to support
 // loadbalancing via Cilium LB IPAM + BGP Control Plane.
 func (l *loadbalancers) createSharedIP(ctx context.Context, nodes []*v1.Node) (string, error) {
 	if Options.BGPNodeSelector == "" {
-		return "", noBGPSelector
+		return "", errNoBGPSelector
 	}
 
 	ipHolder, err := l.ensureIPHolder(ctx)
@@ -81,7 +116,7 @@ func (l *loadbalancers) createSharedIP(ctx context.Context, nodes []*v1.Node) (s
 // by Cilium LB IPAM, removing it from the ip-holder
 func (l *loadbalancers) deleteSharedIP(ctx context.Context, service *v1.Service) error {
 	if Options.BGPNodeSelector == "" {
-		return errors.New("no BGP node label set to configure IP sharing")
+		return errNoBGPSelector
 	}
 	err := l.retrieveKubeClient()
 	if err != nil {
@@ -245,3 +280,76 @@ func (l *loadbalancers) getCiliumLBIPPool(ctx context.Context, service *v1.Servi
 		metav1.GetOptions{},
 	)
 }
+
+// NOTE: Cilium CRDs must be installed for this to work
+func (l *loadbalancers) ensureCiliumBGPPeeringPolicy(ctx context.Context) error {
+	if Options.BGPNodeSelector == "" {
+		return errNoBGPSelector
+	}
+	regionID, ok := regionIDMap[l.zone]
+	if !ok {
+		return fmt.Errorf("unsupported region for BGP: %s", l.zone)
+	}
+	if err := l.retrieveCiliumClientset(); err != nil {
+		return err
+	}
+	// check if policy already exists
+	policy, err := l.ciliumClient.CiliumBGPPeeringPolicies().Get(ctx, ciliumBGPPeeringPolicyName, metav1.GetOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		klog.Infof("Failed to get CiliumBGPPeeringPolicy: %s", err.Error())
+		return err
+	}
+	// if the CiliumBGPPeeringPolicy doesn't exist, it's not nil, just empty
+	if policy != nil && policy.Name != "" {
+		return nil
+	}
+
+	// otherwise create it
+	kv := strings.Split(Options.BGPNodeSelector, "=")
+	if len(kv) != 2 {
+		return fmt.Errorf("invalid node selector %s", Options.BGPNodeSelector)
+	}
+	ciliumBGPPeeringPolicy := &v2alpha1.CiliumBGPPeeringPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ciliumBGPPeeringPolicyName,
+		},
+		Spec: v2alpha1.CiliumBGPPeeringPolicySpec{
+			NodeSelector: &slimv1.LabelSelector{MatchLabels: map[string]string{kv[0]: kv[1]}},
+			VirtualRouters: []v2alpha1.CiliumBGPVirtualRouter{{
+				LocalASN:      65001,
+				ExportPodCIDR: Pointer(true),
+				ServiceSelector: &slimv1.LabelSelector{
+					MatchExpressions: []slimv1.LabelSelectorRequirement{{
+						Key:      "somekey",
+						Operator: slimv1.LabelSelectorOpNotIn,
+						Values:   []string{"never-used-value"},
+					}},
+				},
+			}},
+		},
+	}
+	for i := 1; i <= 4; i++ {
+		neighbor := v2alpha1.CiliumBGPNeighbor{
+			PeerAddress:             fmt.Sprintf("2600:3c0f:%d:34::%d/64", regionID, i),
+			PeerASN:                 65000,
+			EBGPMultihopTTL:         Pointer(int32(10)),
+			ConnectRetryTimeSeconds: Pointer(int32(5)),
+			HoldTimeSeconds:         Pointer(int32(9)),
+			KeepAliveTimeSeconds:    Pointer(int32(3)),
+			AdvertisedPathAttributes: []v2alpha1.CiliumBGPPathAttributes{
+				{
+					SelectorType: "CiliumLoadBalancerIPPool",
+					Communities: &v2alpha1.BGPCommunities{
+						Standard: []v2alpha1.BGPStandardCommunity{"65000:1", "65000:2"},
+					},
+				},
+			},
+		}
+		ciliumBGPPeeringPolicy.Spec.VirtualRouters[0].Neighbors = append(ciliumBGPPeeringPolicy.Spec.VirtualRouters[0].Neighbors, neighbor)
+	}
+
+	klog.Info("Creating CiliumBGPPeeringPolicy")
+	_, err = l.ciliumClient.CiliumBGPPeeringPolicies().Create(ctx, ciliumBGPPeeringPolicy, metav1.CreateOptions{})
+
+	return err
+}

cloud/linode/cilium_loadbalancers_test.go

@@ -21,6 +21,7 @@ import (
 )
 
 var (
+	zone  = "us-ord"
 	nodes = []*v1.Node{
 		{
 			ObjectMeta: metav1.ObjectMeta{
@@ -68,6 +69,10 @@ func TestCiliumCCMLoadBalancers(t *testing.T) {
 			name: "Create Cilium Load Balancer Without BGP Node Labels specified",
 			f:    testNoBGPNodeLabel,
 		},
+		{
+			name: "Create Cilium Load Balancer with unsupported region",
+			f:    testUnsupportedRegion,
+		},
 		{
 			name: "Create Cilium Load Balancer With explicit loadBalancerClass and existing IP holder nanode",
 			f:    testCreateWithExistingIPHolder,
@@ -155,11 +160,28 @@ func testNoBGPNodeLabel(t *testing.T, mc *mocks.MockClient) {
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, nodeBalancerLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, nodeBalancerLBType}
+
+	lbStatus, err := lb.EnsureLoadBalancer(context.TODO(), "linodelb", svc, nodes)
+	if !errors.Is(err, errNoBGPSelector) {
+		t.Fatalf("expected %v, got %v... %s", errNoBGPSelector, err, Options.BGPNodeSelector)
+	}
+	if lbStatus != nil {
+		t.Fatalf("expected a nil lbStatus, got %v", lbStatus)
+	}
+}
+
+func testUnsupportedRegion(t *testing.T, mc *mocks.MockClient) {
+	Options.BGPNodeSelector = "cilium-bgp-peering=true"
+	svc := createTestService(Pointer(ciliumLBType))
+	kubeClient := fake.NewSimpleClientset()
+	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
+	addService(t, kubeClient, svc)
+	lb := &loadbalancers{mc, "us-foobar", kubeClient, ciliumClient, nodeBalancerLBType}
 
 	lbStatus, err := lb.EnsureLoadBalancer(context.TODO(), "linodelb", svc, nodes)
-	if !errors.Is(err, noBGPSelector) {
-		t.Fatalf("expected %v, got %v", noBGPSelector, err)
+	if err == nil {
+		t.Fatal("expected nil error")
 	}
 	if lbStatus != nil {
 		t.Fatalf("expected a nil lbStatus, got %v", lbStatus)
@@ -173,7 +195,7 @@ func testCreateWithExistingIPHolder(t *testing.T, mc *mocks.MockClient) {
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, nodeBalancerLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, nodeBalancerLBType}
 
 	filter := map[string]string{"label": ipHolderLabel}
 	rawFilter, _ := json.Marshal(filter)
@@ -217,7 +239,7 @@ func testCreateWithDefaultLBCilium(t *testing.T, mc *mocks.MockClient) {
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, ciliumLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, ciliumLBType}
 
 	filter := map[string]string{"label": ipHolderLabel}
 	rawFilter, _ := json.Marshal(filter)
@@ -261,7 +283,7 @@ func testCreateWithDefaultLBCiliumNodebalancher(t *testing.T, mc *mocks.MockClie
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, ciliumLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, ciliumLBType}
 
 	mc.EXPECT().CreateNodeBalancer(gomock.Any(), gomock.Any()).Times(1).Return(&linodego.NodeBalancer{
 		ID:       12345,
@@ -287,7 +309,7 @@ func testCreateWithNoExistingIPHolder(t *testing.T, mc *mocks.MockClient) {
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, ciliumLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, ciliumLBType}
 
 	filter := map[string]string{"label": ipHolderLabel}
 	rawFilter, _ := json.Marshal(filter)
@@ -333,7 +355,7 @@ func testEnsureCiliumLoadBalancerDeleted(t *testing.T, mc *mocks.MockClient) {
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
 	addNodes(t, kubeClient, nodes)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, nodeBalancerLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, nodeBalancerLBType}
 
 	dummySharedIP := "45.76.101.26"
 	svc.Status.LoadBalancer = v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: dummySharedIP}}}
@@ -358,7 +380,7 @@ func testDeleteNBWithDefaultLBCilium(t *testing.T, mc *mocks.MockClient) {
 	kubeClient := fake.NewSimpleClientset()
 	ciliumClient := &fakev2alpha1.FakeCiliumV2alpha1{Fake: &kubeClient.Fake}
 	addService(t, kubeClient, svc)
-	lb := &loadbalancers{mc, "us-west", kubeClient, ciliumClient, ciliumLBType}
+	lb := &loadbalancers{mc, zone, kubeClient, ciliumClient, ciliumLBType}
 
 	dummySharedIP := "45.76.101.26"
 	svc.Status.LoadBalancer = v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: dummySharedIP}}}

cloud/linode/client/client.go

@@ -12,10 +12,10 @@ type Client interface {
 	GetInstance(context.Context, int) (*linodego.Instance, error)
 	ListInstances(context.Context, *linodego.ListOptions) ([]linodego.Instance, error)
 	CreateInstance(ctx context.Context, opts linodego.InstanceCreateOptions) (*linodego.Instance, error)
+
 	GetInstanceIPAddresses(context.Context, int) (*linodego.InstanceIPAddressResponse, error)
 	AddInstanceIPAddress(ctx context.Context, linodeID int, public bool) (*linodego.InstanceIP, error)
 	DeleteInstanceIPAddress(ctx context.Context, linodeID int, ipAddress string) error
-
 	ShareIPAddresses(ctx context.Context, opts linodego.IPAddressesShareOptions) error
 
 	UpdateInstanceConfigInterface(context.Context, int, int, int, linodego.InstanceConfigInterfaceUpdateOptions) (*linodego.InstanceConfigInterface, error)

cloud/linode/cloud.go

@@ -17,12 +17,16 @@ import (
 
 const (
 	// The name of this cloudprovider
-	ProviderName   = "linode"
-	accessTokenEnv = "LINODE_API_TOKEN"
-	regionEnv      = "LINODE_REGION"
-	urlEnv         = "LINODE_URL"
+	ProviderName       = "linode"
+	accessTokenEnv     = "LINODE_API_TOKEN"
+	regionEnv          = "LINODE_REGION"
+	urlEnv             = "LINODE_URL"
+	ciliumLBType       = "cilium-bgp"
+	nodeBalancerLBType = "nodebalancer"
 )
 
+var supportedLoadBalancerTypes = []string{ciliumLBType, nodeBalancerLBType}
+
 // Options is a configuration object for this cloudprovider implementation.
 // We expect it to be initialized with flags external to this package, likely in
 // main.go
@@ -35,8 +39,6 @@ var Options struct {
 	BGPNodeSelector       string
 }
 
-var supportedLoadBalancerTypes = []string{ciliumLBType, nodeBalancerLBType}
-
 // vpcDetails is set when VPCName options flag is set.
 // We use it to list instances running within the VPC if set
 type vpcDetails struct {

cloud/linode/loadbalancers.go

@@ -29,11 +29,6 @@ import (
 	"github.com/linode/linode-cloud-controller-manager/sentry"
 )
 
-const (
-	ciliumLBType       = "cilium-bgp"
-	nodeBalancerLBType = "nodebalancer"
-)
-
 var errNoNodesAvailable = errors.New("no nodes available for nodebalancer")
 
 type lbNotFoundError struct {
@@ -214,6 +209,11 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 	if l.isCiliumLBType(service) {
 		klog.Infof("handling LoadBalancer Service %s as %s", serviceNn, ciliumLBClass)
 
+		if err = l.ensureCiliumBGPPeeringPolicy(ctx); err != nil {
+			klog.Infof("Failed to ensure CiliumBGPPeeringPolicy: %v", err)
+			return nil, err
+		}
+
 		// check for existing CiliumLoadBalancerIPPool for service
 		pool, err := l.getCiliumLBIPPool(ctx, service)
 		// if the CiliumLoadBalancerIPPool doesn't exist, it's not nil, just empty
@@ -226,20 +226,17 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 		}
 		if err != nil && !k8serrors.IsNotFound(err) {
 			klog.Infof("Failed to get CiliumLoadBalancerIPPool: %s", err.Error())
-			sentry.CaptureError(ctx, err)
 			return nil, err
 		}
 
 		// CiliumLoadBalancerIPPool does not yet exist for the service
 		var sharedIP string
 		if sharedIP, err = l.createSharedIP(ctx, nodes); err != nil {
 			klog.Errorf("Failed to request shared instance IP: %s", err.Error())
-			sentry.CaptureError(ctx, err)
 			return nil, err
 		}
 		if _, err = l.createCiliumLBIPPool(ctx, service, sharedIP); err != nil {
 			klog.Infof("Failed to create CiliumLoadBalancerIPPool: %s", err.Error())
-			sentry.CaptureError(ctx, err)
 			return nil, err
 		}
 
@@ -505,14 +502,11 @@ func (l *loadbalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa
 	if l.isCiliumLBType(service) {
 		klog.Infof("handling LoadBalancer Service %s/%s as %s", service.Namespace, service.Name, ciliumLBClass)
 		if err := l.deleteSharedIP(ctx, service); err != nil {
-			klog.Infof("Failed to delete shared instance IP")
-			sentry.CaptureError(ctx, err)
 			return err
 		}
 		// delete CiliumLoadBalancerIPPool for service
 		if err := l.deleteCiliumLBIPPool(ctx, service); err != nil && !k8serrors.IsNotFound(err) {
 			klog.Infof("Failed to delete CiliumLoadBalancerIPPool")
-			sentry.CaptureError(ctx, err)
 			return err
 		}