From 0a3730f0c41aa717f22d897e67dd0ba71c1aec82 Mon Sep 17 00:00:00 2001 From: Ashley Dumaine Date: Fri, 8 Mar 2024 15:10:23 -0500 Subject: [PATCH] refactor into distinct packages to prep for adding firewall support to Nodes --- cloud/annotations/annotations.go | 34 ++ cloud/linode/annotations.go | 34 -- cloud/linode/{ => client}/client.go | 9 +- cloud/linode/client/mock_client_test.go | 332 +++++++++++++++ cloud/linode/cloud.go | 6 +- cloud/linode/firewall/firewalls.go | 457 +++++++++++++++++++++ cloud/linode/instances.go | 10 +- cloud/linode/loadbalancers.go | 512 +++--------------------- cloud/linode/loadbalancers_test.go | 204 +++++----- cloud/linode/node_controller.go | 13 +- 10 files changed, 1002 insertions(+), 609 deletions(-) create mode 100644 cloud/annotations/annotations.go delete mode 100644 cloud/linode/annotations.go rename cloud/linode/{ => client}/client.go (89%) create mode 100644 cloud/linode/client/mock_client_test.go create mode 100644 cloud/linode/firewall/firewalls.go diff --git a/cloud/annotations/annotations.go b/cloud/annotations/annotations.go new file mode 100644 index 00000000..aa390154 --- /dev/null +++ b/cloud/annotations/annotations.go @@ -0,0 +1,34 @@ +package annotations + +const ( + // AnnLinodeDefaultProtocol is the annotation used to specify the default protocol + // for Linode load balancers. Options are tcp, http and https. Defaults to tcp. + AnnLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" + AnnLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" + AnnLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" + + AnnLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" + AnnLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" + AnnLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type" + + AnnLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval" + AnnLinodeHealthCheckTimeout = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout" + AnnLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts" + AnnLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive" + + // AnnLinodeThrottle is the annotation specifying the value of the Client Connection + // Throttle, which limits the number of subsequent new connections per second from the + // same client IP. Options are a number between 1-20, or 0 to disable. Defaults to 20. + AnnLinodeThrottle = "service.beta.kubernetes.io/linode-loadbalancer-throttle" + + AnnLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve" + AnnLinodeNodeBalancerID = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id" + + AnnLinodeHostnameOnlyIngress = "service.beta.kubernetes.io/linode-loadbalancer-hostname-only-ingress" + AnnLinodeLoadBalancerTags = "service.beta.kubernetes.io/linode-loadbalancer-tags" + AnnLinodeCloudFirewallID = "service.beta.kubernetes.io/linode-loadbalancer-firewall-id" + AnnLinodeCloudFirewallACL = "service.beta.kubernetes.io/linode-loadbalancer-firewall-acl" + + AnnLinodeNodePrivateIP = "node.k8s.linode.com/private-ip" + AnnLinodeHostUUID = "node.k8s.linode.com/host-uuid" +) diff --git a/cloud/linode/annotations.go b/cloud/linode/annotations.go deleted file mode 100644 index 18c0f874..00000000 --- a/cloud/linode/annotations.go +++ /dev/null @@ -1,34 +0,0 @@ -package linode - -const ( - // annLinodeDefaultProtocol is the annotation used to specify the default protocol - // for Linode load balancers. Options are tcp, http and https. Defaults to tcp. - annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" - annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" - annLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" - - annLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" - annLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" - annLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type" - - annLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval" - annLinodeHealthCheckTimeout = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout" - annLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts" - annLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive" - - // annLinodeThrottle is the annotation specifying the value of the Client Connection - // Throttle, which limits the number of subsequent new connections per second from the - // same client IP. Options are a number between 1-20, or 0 to disable. Defaults to 20. - annLinodeThrottle = "service.beta.kubernetes.io/linode-loadbalancer-throttle" - - annLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve" - annLinodeNodeBalancerID = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id" - - annLinodeHostnameOnlyIngress = "service.beta.kubernetes.io/linode-loadbalancer-hostname-only-ingress" - annLinodeLoadBalancerTags = "service.beta.kubernetes.io/linode-loadbalancer-tags" - annLinodeCloudFirewallID = "service.beta.kubernetes.io/linode-loadbalancer-firewall-id" - annLinodeCloudFirewallACL = "service.beta.kubernetes.io/linode-loadbalancer-firewall-acl" - - annLinodeNodePrivateIP = "node.k8s.linode.com/private-ip" - annLinodeHostUUID = "node.k8s.linode.com/host-uuid" -) diff --git a/cloud/linode/client.go b/cloud/linode/client/client.go similarity index 89% rename from cloud/linode/client.go rename to cloud/linode/client/client.go index a2fcde09..3b9bb74d 100644 --- a/cloud/linode/client.go +++ b/cloud/linode/client/client.go @@ -1,6 +1,6 @@ -package linode +package client -//go:generate go run github.com/golang/mock/mockgen -destination mock_client_test.go -package linode github.com/linode/linode-cloud-controller-manager/cloud/linode Client +//go:generate go run github.com/golang/mock/mockgen -destination mock_client_test.go -package client github.com/linode/linode-cloud-controller-manager/cloud/linode/client Client import ( "context" @@ -39,9 +39,10 @@ type Client interface { // linodego.Client implements Client var _ Client = (*linodego.Client)(nil) -func newLinodeClient(token, ua, apiURL string) (*linodego.Client, error) { +// New creates a new linode client with a given token, userAgent, and API URL +func New(token, userAgent, apiURL string) (*linodego.Client, error) { linodeClient := linodego.NewClient(nil) - linodeClient.SetUserAgent(ua) + linodeClient.SetUserAgent(userAgent) linodeClient.SetToken(token) // Validate apiURL diff --git a/cloud/linode/client/mock_client_test.go b/cloud/linode/client/mock_client_test.go new file mode 100644 index 00000000..c7535897 --- /dev/null +++ b/cloud/linode/client/mock_client_test.go @@ -0,0 +1,332 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/linode/linode-cloud-controller-manager/cloud/linode/client (interfaces: Client) + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + linodego "github.com/linode/linodego" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// CreateFirewall mocks base method. +func (m *MockClient) CreateFirewall(arg0 context.Context, arg1 linodego.FirewallCreateOptions) (*linodego.Firewall, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFirewall", arg0, arg1) + ret0, _ := ret[0].(*linodego.Firewall) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFirewall indicates an expected call of CreateFirewall. +func (mr *MockClientMockRecorder) CreateFirewall(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFirewall", reflect.TypeOf((*MockClient)(nil).CreateFirewall), arg0, arg1) +} + +// CreateFirewallDevice mocks base method. +func (m *MockClient) CreateFirewallDevice(arg0 context.Context, arg1 int, arg2 linodego.FirewallDeviceCreateOptions) (*linodego.FirewallDevice, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFirewallDevice", arg0, arg1, arg2) + ret0, _ := ret[0].(*linodego.FirewallDevice) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFirewallDevice indicates an expected call of CreateFirewallDevice. +func (mr *MockClientMockRecorder) CreateFirewallDevice(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFirewallDevice", reflect.TypeOf((*MockClient)(nil).CreateFirewallDevice), arg0, arg1, arg2) +} + +// CreateNodeBalancer mocks base method. +func (m *MockClient) CreateNodeBalancer(arg0 context.Context, arg1 linodego.NodeBalancerCreateOptions) (*linodego.NodeBalancer, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNodeBalancer", arg0, arg1) + ret0, _ := ret[0].(*linodego.NodeBalancer) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateNodeBalancer indicates an expected call of CreateNodeBalancer. +func (mr *MockClientMockRecorder) CreateNodeBalancer(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNodeBalancer", reflect.TypeOf((*MockClient)(nil).CreateNodeBalancer), arg0, arg1) +} + +// CreateNodeBalancerConfig mocks base method. +func (m *MockClient) CreateNodeBalancerConfig(arg0 context.Context, arg1 int, arg2 linodego.NodeBalancerConfigCreateOptions) (*linodego.NodeBalancerConfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNodeBalancerConfig", arg0, arg1, arg2) + ret0, _ := ret[0].(*linodego.NodeBalancerConfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateNodeBalancerConfig indicates an expected call of CreateNodeBalancerConfig. +func (mr *MockClientMockRecorder) CreateNodeBalancerConfig(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNodeBalancerConfig", reflect.TypeOf((*MockClient)(nil).CreateNodeBalancerConfig), arg0, arg1, arg2) +} + +// DeleteFirewall mocks base method. +func (m *MockClient) DeleteFirewall(arg0 context.Context, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteFirewall", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteFirewall indicates an expected call of DeleteFirewall. +func (mr *MockClientMockRecorder) DeleteFirewall(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteFirewall", reflect.TypeOf((*MockClient)(nil).DeleteFirewall), arg0, arg1) +} + +// DeleteFirewallDevice mocks base method. +func (m *MockClient) DeleteFirewallDevice(arg0 context.Context, arg1, arg2 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteFirewallDevice", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteFirewallDevice indicates an expected call of DeleteFirewallDevice. +func (mr *MockClientMockRecorder) DeleteFirewallDevice(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteFirewallDevice", reflect.TypeOf((*MockClient)(nil).DeleteFirewallDevice), arg0, arg1, arg2) +} + +// DeleteNodeBalancer mocks base method. +func (m *MockClient) DeleteNodeBalancer(arg0 context.Context, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteNodeBalancer", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteNodeBalancer indicates an expected call of DeleteNodeBalancer. +func (mr *MockClientMockRecorder) DeleteNodeBalancer(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNodeBalancer", reflect.TypeOf((*MockClient)(nil).DeleteNodeBalancer), arg0, arg1) +} + +// DeleteNodeBalancerConfig mocks base method. +func (m *MockClient) DeleteNodeBalancerConfig(arg0 context.Context, arg1, arg2 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteNodeBalancerConfig", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteNodeBalancerConfig indicates an expected call of DeleteNodeBalancerConfig. +func (mr *MockClientMockRecorder) DeleteNodeBalancerConfig(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNodeBalancerConfig", reflect.TypeOf((*MockClient)(nil).DeleteNodeBalancerConfig), arg0, arg1, arg2) +} + +// GetFirewall mocks base method. +func (m *MockClient) GetFirewall(arg0 context.Context, arg1 int) (*linodego.Firewall, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFirewall", arg0, arg1) + ret0, _ := ret[0].(*linodego.Firewall) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFirewall indicates an expected call of GetFirewall. +func (mr *MockClientMockRecorder) GetFirewall(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirewall", reflect.TypeOf((*MockClient)(nil).GetFirewall), arg0, arg1) +} + +// GetInstance mocks base method. +func (m *MockClient) GetInstance(arg0 context.Context, arg1 int) (*linodego.Instance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInstance", arg0, arg1) + ret0, _ := ret[0].(*linodego.Instance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInstance indicates an expected call of GetInstance. +func (mr *MockClientMockRecorder) GetInstance(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstance", reflect.TypeOf((*MockClient)(nil).GetInstance), arg0, arg1) +} + +// GetInstanceIPAddresses mocks base method. +func (m *MockClient) GetInstanceIPAddresses(arg0 context.Context, arg1 int) (*linodego.InstanceIPAddressResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInstanceIPAddresses", arg0, arg1) + ret0, _ := ret[0].(*linodego.InstanceIPAddressResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInstanceIPAddresses indicates an expected call of GetInstanceIPAddresses. +func (mr *MockClientMockRecorder) GetInstanceIPAddresses(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstanceIPAddresses", reflect.TypeOf((*MockClient)(nil).GetInstanceIPAddresses), arg0, arg1) +} + +// GetNodeBalancer mocks base method. +func (m *MockClient) GetNodeBalancer(arg0 context.Context, arg1 int) (*linodego.NodeBalancer, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNodeBalancer", arg0, arg1) + ret0, _ := ret[0].(*linodego.NodeBalancer) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNodeBalancer indicates an expected call of GetNodeBalancer. +func (mr *MockClientMockRecorder) GetNodeBalancer(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeBalancer", reflect.TypeOf((*MockClient)(nil).GetNodeBalancer), arg0, arg1) +} + +// ListFirewallDevices mocks base method. +func (m *MockClient) ListFirewallDevices(arg0 context.Context, arg1 int, arg2 *linodego.ListOptions) ([]linodego.FirewallDevice, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListFirewallDevices", arg0, arg1, arg2) + ret0, _ := ret[0].([]linodego.FirewallDevice) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListFirewallDevices indicates an expected call of ListFirewallDevices. +func (mr *MockClientMockRecorder) ListFirewallDevices(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFirewallDevices", reflect.TypeOf((*MockClient)(nil).ListFirewallDevices), arg0, arg1, arg2) +} + +// ListInstances mocks base method. +func (m *MockClient) ListInstances(arg0 context.Context, arg1 *linodego.ListOptions) ([]linodego.Instance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListInstances", arg0, arg1) + ret0, _ := ret[0].([]linodego.Instance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListInstances indicates an expected call of ListInstances. +func (mr *MockClientMockRecorder) ListInstances(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInstances", reflect.TypeOf((*MockClient)(nil).ListInstances), arg0, arg1) +} + +// ListNodeBalancerConfigs mocks base method. +func (m *MockClient) ListNodeBalancerConfigs(arg0 context.Context, arg1 int, arg2 *linodego.ListOptions) ([]linodego.NodeBalancerConfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListNodeBalancerConfigs", arg0, arg1, arg2) + ret0, _ := ret[0].([]linodego.NodeBalancerConfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNodeBalancerConfigs indicates an expected call of ListNodeBalancerConfigs. +func (mr *MockClientMockRecorder) ListNodeBalancerConfigs(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodeBalancerConfigs", reflect.TypeOf((*MockClient)(nil).ListNodeBalancerConfigs), arg0, arg1, arg2) +} + +// ListNodeBalancerFirewalls mocks base method. +func (m *MockClient) ListNodeBalancerFirewalls(arg0 context.Context, arg1 int, arg2 *linodego.ListOptions) ([]linodego.Firewall, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListNodeBalancerFirewalls", arg0, arg1, arg2) + ret0, _ := ret[0].([]linodego.Firewall) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNodeBalancerFirewalls indicates an expected call of ListNodeBalancerFirewalls. +func (mr *MockClientMockRecorder) ListNodeBalancerFirewalls(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodeBalancerFirewalls", reflect.TypeOf((*MockClient)(nil).ListNodeBalancerFirewalls), arg0, arg1, arg2) +} + +// ListNodeBalancers mocks base method. +func (m *MockClient) ListNodeBalancers(arg0 context.Context, arg1 *linodego.ListOptions) ([]linodego.NodeBalancer, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListNodeBalancers", arg0, arg1) + ret0, _ := ret[0].([]linodego.NodeBalancer) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNodeBalancers indicates an expected call of ListNodeBalancers. +func (mr *MockClientMockRecorder) ListNodeBalancers(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodeBalancers", reflect.TypeOf((*MockClient)(nil).ListNodeBalancers), arg0, arg1) +} + +// RebuildNodeBalancerConfig mocks base method. +func (m *MockClient) RebuildNodeBalancerConfig(arg0 context.Context, arg1, arg2 int, arg3 linodego.NodeBalancerConfigRebuildOptions) (*linodego.NodeBalancerConfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RebuildNodeBalancerConfig", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*linodego.NodeBalancerConfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RebuildNodeBalancerConfig indicates an expected call of RebuildNodeBalancerConfig. +func (mr *MockClientMockRecorder) RebuildNodeBalancerConfig(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebuildNodeBalancerConfig", reflect.TypeOf((*MockClient)(nil).RebuildNodeBalancerConfig), arg0, arg1, arg2, arg3) +} + +// UpdateFirewallRules mocks base method. +func (m *MockClient) UpdateFirewallRules(arg0 context.Context, arg1 int, arg2 linodego.FirewallRuleSet) (*linodego.FirewallRuleSet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateFirewallRules", arg0, arg1, arg2) + ret0, _ := ret[0].(*linodego.FirewallRuleSet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateFirewallRules indicates an expected call of UpdateFirewallRules. +func (mr *MockClientMockRecorder) UpdateFirewallRules(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFirewallRules", reflect.TypeOf((*MockClient)(nil).UpdateFirewallRules), arg0, arg1, arg2) +} + +// UpdateNodeBalancer mocks base method. +func (m *MockClient) UpdateNodeBalancer(arg0 context.Context, arg1 int, arg2 linodego.NodeBalancerUpdateOptions) (*linodego.NodeBalancer, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateNodeBalancer", arg0, arg1, arg2) + ret0, _ := ret[0].(*linodego.NodeBalancer) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateNodeBalancer indicates an expected call of UpdateNodeBalancer. +func (mr *MockClientMockRecorder) UpdateNodeBalancer(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNodeBalancer", reflect.TypeOf((*MockClient)(nil).UpdateNodeBalancer), arg0, arg1, arg2) +} diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 885b80a6..123c2039 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -9,6 +9,8 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/informers" cloudprovider "k8s.io/cloud-provider" + + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" ) const ( @@ -28,7 +30,7 @@ var Options struct { } type linodeCloud struct { - client Client + client client.Client instances cloudprovider.InstancesV2 loadbalancers cloudprovider.LoadBalancer } @@ -56,7 +58,7 @@ func newCloud() (cloudprovider.Interface, error) { url := os.Getenv(urlEnv) ua := fmt.Sprintf("linode-cloud-controller-manager %s", linodego.DefaultUserAgent) - linodeClient, err := newLinodeClient(apiToken, ua, url) + linodeClient, err := client.New(apiToken, ua, url) if err != nil { return nil, fmt.Errorf("client was not created succesfully: %w", err) } diff --git a/cloud/linode/firewall/firewalls.go b/cloud/linode/firewall/firewalls.go new file mode 100644 index 00000000..6ddb8295 --- /dev/null +++ b/cloud/linode/firewall/firewalls.go @@ -0,0 +1,457 @@ +package firewall + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/exp/slices" + + "github.com/linode/linodego" + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" +) + +const ( + maxFirewallRuleLabelLen = 32 + maxIPsPerFirewall = 255 + maxRulesPerFirewall = 25 +) + +var ( + ErrTooManyIPs = errors.New("too many IPs in this ACL, will exceed rules per firewall limit") + ErrTooManyNBFirewalls = errors.New("too many firewalls attached to a nodebalancer") + ErrInvalidFWConfig = errors.New("specify either an allowList or a denyList for a firewall") +) + +type LinodeClient struct { + Client client.Client +} + +type aclConfig struct { + AllowList *linodego.NetworkAddresses `json:"allowList"` + DenyList *linodego.NetworkAddresses `json:"denyList"` +} + +func (l *LinodeClient) CreateFirewall(ctx context.Context, opts linodego.FirewallCreateOptions) (fw *linodego.Firewall, err error) { + return l.Client.CreateFirewall(ctx, opts) +} + +func (l *LinodeClient) DeleteFirewall(ctx context.Context, firewall *linodego.Firewall) error { + return l.Client.DeleteFirewall(ctx, firewall.ID) +} + +func ipsChanged(ips *linodego.NetworkAddresses, rules []linodego.FirewallRule) bool { + var ruleIPv4s []string + var ruleIPv6s []string + + for _, rule := range rules { + if rule.Addresses.IPv4 != nil { + ruleIPv4s = append(ruleIPv4s, *rule.Addresses.IPv4...) + } + if rule.Addresses.IPv6 != nil { + ruleIPv6s = append(ruleIPv6s, *rule.Addresses.IPv6...) + } + } + + if len(ruleIPv4s) > 0 && ips.IPv4 == nil { + return true + } + + if len(ruleIPv6s) > 0 && ips.IPv6 == nil { + return true + } + + if ips.IPv4 != nil { + for _, ipv4 := range *ips.IPv4 { + if !slices.Contains(ruleIPv4s, ipv4) { + return true + } + } + } + + if ips.IPv6 != nil { + for _, ipv6 := range *ips.IPv6 { + if !slices.Contains(ruleIPv6s, ipv6) { + return true + } + } + } + + return false +} + +// ruleChanged takes an old FirewallRuleSet and new aclConfig and returns if +// the IPs of the FirewallRuleSet would be changed with the new ACL Config +func ruleChanged(old linodego.FirewallRuleSet, newACL aclConfig) bool { + var ips *linodego.NetworkAddresses + if newACL.AllowList != nil { + // this is a allowList, this means that the rules should have `DROP` as inboundpolicy + if old.InboundPolicy != "DROP" { + return true + } + if (newACL.AllowList.IPv4 != nil || newACL.AllowList.IPv6 != nil) && len(old.Inbound) == 0 { + return true + } + ips = newACL.AllowList + } + + if newACL.DenyList != nil { + if old.InboundPolicy != "ACCEPT" { + return true + } + + if (newACL.DenyList.IPv4 != nil || newACL.DenyList.IPv6 != nil) && len(old.Inbound) == 0 { + return true + } + ips = newACL.DenyList + } + + return ipsChanged(ips, old.Inbound) +} + +func chunkIPs(ips []string) [][]string { + chunks := [][]string{} + ipCount := len(ips) + + // If the number of IPs is less than or equal to maxIPsPerFirewall, + // return a single chunk containing all IPs. + if ipCount <= maxIPsPerFirewall { + return [][]string{ips} + } + + // Otherwise, break the IPs into chunks with maxIPsPerFirewall IPs per chunk. + chunkCount := 0 + for ipCount > maxIPsPerFirewall { + start := chunkCount * maxIPsPerFirewall + end := (chunkCount + 1) * maxIPsPerFirewall + chunks = append(chunks, ips[start:end]) + chunkCount++ + ipCount -= maxIPsPerFirewall + } + + // Append the remaining IPs as a chunk. + chunks = append(chunks, ips[chunkCount*maxIPsPerFirewall:]) + + return chunks +} + +// processACL takes the IPs, aclType, label etc and formats them into the passed linodego.FirewallCreateOptions pointer. +func processACL(fwcreateOpts *linodego.FirewallCreateOptions, aclType, label, svcName, ports string, ips linodego.NetworkAddresses) error { + ruleLabel := fmt.Sprintf("%s-%s", aclType, svcName) + if len(ruleLabel) > maxFirewallRuleLabelLen { + newLabel := ruleLabel[0:maxFirewallRuleLabelLen] + klog.Infof("Firewall label '%s' is too long. Stripping to '%s'", ruleLabel, newLabel) + ruleLabel = newLabel + } + + // Linode has a limitation of firewall rules with a max of 255 IPs per rule + var ipv4s, ipv6s []string // doing this to avoid dereferencing a nil pointer + if ips.IPv6 != nil { + ipv6s = *ips.IPv6 + } + if ips.IPv4 != nil { + ipv4s = *ips.IPv4 + } + + if len(ipv4s)+len(ipv6s) > maxIPsPerFirewall { + ipv4chunks := chunkIPs(ipv4s) + for i, chunk := range ipv4chunks { + v4chunk := chunk + fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ + Action: aclType, + Label: ruleLabel, + Description: fmt.Sprintf("Rule %d, Created by linode-ccm: %s, for %s", i, label, svcName), + Protocol: linodego.TCP, // Nodebalancers support only TCP. + Ports: ports, + Addresses: linodego.NetworkAddresses{IPv4: &v4chunk}, + }) + } + + ipv6chunks := chunkIPs(ipv6s) + for i, chunk := range ipv6chunks { + v6chunk := chunk + fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ + Action: aclType, + Label: ruleLabel, + Description: fmt.Sprintf("Rule %d, Created by linode-ccm: %s, for %s", i, label, svcName), + Protocol: linodego.TCP, // Nodebalancers support only TCP. + Ports: ports, + Addresses: linodego.NetworkAddresses{IPv6: &v6chunk}, + }) + } + } else { + fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ + Action: aclType, + Label: ruleLabel, + Description: fmt.Sprintf("Created by linode-ccm: %s, for %s", label, svcName), + Protocol: linodego.TCP, // Nodebalancers support only TCP. + Ports: ports, + Addresses: ips, + }) + } + + fwcreateOpts.Rules.OutboundPolicy = "ACCEPT" + if aclType == "ACCEPT" { + // if an allowlist is present, we drop everything else. + fwcreateOpts.Rules.InboundPolicy = "DROP" + } else { + // if a denylist is present, we accept everything else. + fwcreateOpts.Rules.InboundPolicy = "ACCEPT" + } + + if len(fwcreateOpts.Rules.Inbound) > maxRulesPerFirewall { + return ErrTooManyIPs + } + return nil +} + +// UpdateNodeBalancerFirewall reconciles the firewall attached to the nodebalancer +// +// This function does the following +// 1. If a firewallID annotation is present, it checks if the nodebalancer has a firewall attached, and if it matches the annotationID +// a. If the IDs match, nothing to do here. +// b. If they don't match, the nb is attached to the new firewall and removed from the old one. +// 2. If a firewallACL annotation is present, +// a. it checks if the nodebalancer has a firewall attached, if a fw exists, it updates rules +// b. if a fw does not exist, it creates one +// 3. If neither of these annotations are present, +// a. AND if no firewalls are attached to the nodebalancer, nothing to do. +// b. if the NB has ONE firewall attached, remove it from nb, and clean up if nothing else is attached to it +// c. If there are more than one fw attached to it, then its a problem, return an err +// 4. If both these annotations are present, the firewallID takes precedence, and the ACL annotation is ignored. +// +// IF a user creates a fw ID externally, and then switches to using a ACL, the CCM will take over the fw that's attached to the nodebalancer. +func (l *LinodeClient) UpdateNodeBalancerFirewall( + ctx context.Context, + loadBalancerName string, + loadBalancerTags []string, + service *v1.Service, + nb *linodego.NodeBalancer, +) error { + // get the new firewall id from the annotation (if any). + _, fwIDExists := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallID] + if fwIDExists { // If an ID exists, we ignore everything else and handle just that + return l.updateServiceFirewall(ctx, service, nb) + } + + // See if a acl exists + _, fwACLExists := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallACL] + if fwACLExists { // if an ACL exists, but no ID, just update the ACL on the fw. + return l.updateNodeBalancerFirewallWithACL(ctx, loadBalancerName, loadBalancerTags, service, nb) + } + + // No firewall ID or ACL annotation, see if there are firewalls attached to our nb + firewalls, err := l.Client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) + if err != nil { + return err + } + + if len(firewalls) == 0 { + return nil + } + if len(firewalls) > 1 { + klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) + return ErrTooManyNBFirewalls + } + + err = l.Client.DeleteFirewallDevice(ctx, firewalls[0].ID, nb.ID) + if err != nil { + return err + } + // once we delete the device, we should see if there's anything attached to that firewall + devices, err := l.Client.ListFirewallDevices(ctx, firewalls[0].ID, &linodego.ListOptions{}) + if err != nil { + return err + } + + if len(devices) == 0 { + // nothing attached to it, clean it up + return l.Client.DeleteFirewall(ctx, firewalls[0].ID) + } + // else let that firewall linger, don't mess with it. + + return nil +} + +// getNodeBalancerDeviceID gets the deviceID of the nodeBalancer that is attached to the firewall. +func (l *LinodeClient) getNodeBalancerDeviceID(ctx context.Context, firewallID, nbID int) (int, bool, error) { + devices, err := l.Client.ListFirewallDevices(ctx, firewallID, &linodego.ListOptions{}) + if err != nil { + return 0, false, err + } + + if len(devices) == 0 { + return 0, false, nil + } + + for _, device := range devices { + if device.Entity.ID == nbID { + return device.ID, true, nil + } + } + + return 0, false, nil +} + +// Updates a service that has a firewallID annotation set. +// If an annotation is set, and the nodebalancer has a firewall that matches the ID, nothing to do +// If there's more than one firewall attached to the node-balancer, an error is returned as its not a supported use case. +// If there's only one firewall attached and it doesn't match what's in the annotation, the new firewall is attached and the old one removed +func (l *LinodeClient) updateServiceFirewall(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer) error { + var newFirewallID int + var err error + + // See if a firewall is attached to the nodebalancer first. + firewalls, err := l.Client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) + if err != nil { + return err + } + if len(firewalls) > 1 { + klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) + return ErrTooManyNBFirewalls + } + + // get the ID of the firewall that is already attached to the nodeBalancer, if we have one. + var existingFirewallID int + if len(firewalls) == 1 { + existingFirewallID = firewalls[0].ID + } + + fwID := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallID] + newFirewallID, err = strconv.Atoi(fwID) + if err != nil { + return err + } + // if existing firewall and new firewall differs, attach the new firewall and remove the old. + if existingFirewallID != newFirewallID { + // attach new firewall. + _, err = l.Client.CreateFirewallDevice(ctx, newFirewallID, linodego.FirewallDeviceCreateOptions{ + ID: nb.ID, + Type: "nodebalancer", + }) + if err != nil { + return err + } + // remove the existing firewall if it exists + if existingFirewallID != 0 { + deviceID, deviceExists, err := l.getNodeBalancerDeviceID(ctx, existingFirewallID, nb.ID) + if err != nil { + return err + } + + if !deviceExists { + return fmt.Errorf("error in fetching attached nodeBalancer device") + } + + if err = l.Client.DeleteFirewallDevice(ctx, existingFirewallID, deviceID); err != nil { + return err + } + } + } + return nil +} + +func (l *LinodeClient) updateNodeBalancerFirewallWithACL( + ctx context.Context, + loadBalancerName string, + loadBalancerTags []string, + service *v1.Service, + nb *linodego.NodeBalancer, +) error { + // See if a firewall is attached to the nodebalancer first. + firewalls, err := l.Client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) + if err != nil { + return err + } + + switch len(firewalls) { + case 0: + { + // need to create a fw and attach it to our nb + fwcreateOpts, err := CreateFirewallOptsForSvc(loadBalancerName, loadBalancerTags, service) + if err != nil { + return err + } + + fw, err := l.Client.CreateFirewall(ctx, *fwcreateOpts) + if err != nil { + return err + } + // attach new firewall. + if _, err = l.Client.CreateFirewallDevice(ctx, fw.ID, linodego.FirewallDeviceCreateOptions{ + ID: nb.ID, + Type: "nodebalancer", + }); err != nil { + return err + } + } + case 1: + { + // We do not want to get into the complexity of reconciling differences, might as well just pull what's in the svc annotation now and update the fw. + var acl aclConfig + err := json.Unmarshal([]byte(service.GetAnnotations()[annotations.AnnLinodeCloudFirewallACL]), &acl) + if err != nil { + return err + } + + changed := ruleChanged(firewalls[0].Rules, acl) + if !changed { + return nil + } + + fwCreateOpts, err := CreateFirewallOptsForSvc(service.Name, []string{""}, service) + if err != nil { + return err + } + if _, err = l.Client.UpdateFirewallRules(ctx, firewalls[0].ID, fwCreateOpts.Rules); err != nil { + return err + } + } + default: + klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) + return ErrTooManyNBFirewalls + } + return nil +} + +func CreateFirewallOptsForSvc(label string, tags []string, svc *v1.Service) (*linodego.FirewallCreateOptions, error) { + // Fetch acl from annotation + aclString := svc.GetAnnotations()[annotations.AnnLinodeCloudFirewallACL] + fwcreateOpts := linodego.FirewallCreateOptions{ + Label: label, + Tags: tags, + } + servicePorts := make([]string, 0, len(svc.Spec.Ports)) + for _, port := range svc.Spec.Ports { + servicePorts = append(servicePorts, strconv.Itoa(int(port.Port))) + } + + portsString := strings.Join(servicePorts[:], ",") + var acl aclConfig + if err := json.Unmarshal([]byte(aclString), &acl); err != nil { + return nil, err + } + // it is a problem if both are set, or if both are not set + if (acl.AllowList != nil && acl.DenyList != nil) || (acl.AllowList == nil && acl.DenyList == nil) { + return nil, ErrInvalidFWConfig + } + + aclType := "ACCEPT" + allowedIPs := acl.AllowList + if acl.DenyList != nil { + aclType = "DROP" + allowedIPs = acl.DenyList + } + + if err := processACL(&fwcreateOpts, aclType, label, svc.Name, portsString, *allowedIPs); err != nil { + return nil, err + } + return &fwcreateOpts, nil +} diff --git a/cloud/linode/instances.go b/cloud/linode/instances.go index 26c762d0..da38ccb3 100644 --- a/cloud/linode/instances.go +++ b/cloud/linode/instances.go @@ -8,11 +8,13 @@ import ( "sync" "time" - "github.com/linode/linode-cloud-controller-manager/sentry" "github.com/linode/linodego" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" + "github.com/linode/linode-cloud-controller-manager/sentry" ) type nodeCache struct { @@ -24,7 +26,7 @@ type nodeCache struct { // refreshInstances conditionally loads all instances from the Linode API and caches them. // It does not refresh if the last update happened less than `nodeCache.ttl` ago. -func (nc *nodeCache) refreshInstances(ctx context.Context, client Client) error { +func (nc *nodeCache) refreshInstances(ctx context.Context, client client.Client) error { nc.Lock() defer nc.Unlock() @@ -47,12 +49,12 @@ func (nc *nodeCache) refreshInstances(ctx context.Context, client Client) error } type instances struct { - client Client + client client.Client nodeCache *nodeCache } -func newInstances(client Client) *instances { +func newInstances(client client.Client) *instances { timeout := 15 if raw, ok := os.LookupEnv("LINODE_INSTANCE_CACHE_TTL"); ok { if t, _ := strconv.Atoi(raw); t > 0 { diff --git a/cloud/linode/loadbalancers.go b/cloud/linode/loadbalancers.go index 15b3d31d..a661c60e 100644 --- a/cloud/linode/loadbalancers.go +++ b/cloud/linode/loadbalancers.go @@ -12,8 +12,7 @@ import ( "strings" "time" - "golang.org/x/exp/slices" - + "github.com/linode/linodego" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -22,22 +21,13 @@ import ( cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/firewall" "github.com/linode/linode-cloud-controller-manager/sentry" - "github.com/linode/linodego" ) -const ( - maxFirewallRuleLabelLen = 32 - maxIPsPerFirewall = 255 - maxRulesPerFirewall = 25 -) - -var ( - errNoNodesAvailable = errors.New("No nodes available for nodebalancer") - errInvalidFWConfig = errors.New("Specify either an allowList or a denyList for a firewall") - errTooManyFirewalls = errors.New("Too many firewalls attached to a nodebalancer") - errTooManyIPs = errors.New("too many IPs in this ACL, will exceed rules per firewall limit") -) +var errNoNodesAvailable = errors.New("no nodes available for nodebalancer") type lbNotFoundError struct { serviceNn string @@ -52,7 +42,7 @@ func (e lbNotFoundError) Error() string { } type loadbalancers struct { - client Client + client client.Client zone string kubeClient kubernetes.Interface } @@ -71,12 +61,12 @@ type portConfig struct { } // newLoadbalancers returns a cloudprovider.LoadBalancer whose concrete type is a *loadbalancer. -func newLoadbalancers(client Client, zone string) cloudprovider.LoadBalancer { +func newLoadbalancers(client client.Client, zone string) cloudprovider.LoadBalancer { return &loadbalancers{client: client, zone: zone} } func (l *loadbalancers) getNodeBalancerForService(ctx context.Context, service *v1.Service) (*linodego.NodeBalancer, error) { - rawID := service.GetAnnotations()[annLinodeNodeBalancerID] + rawID := service.GetAnnotations()[annotations.AnnLinodeNodeBalancerID] id, idErr := strconv.Atoi(rawID) hasIDAnn := idErr == nil && id != 0 @@ -122,7 +112,7 @@ func (l *loadbalancers) getNodeBalancerByStatus(ctx context.Context, service *v1 func (l *loadbalancers) cleanupOldNodeBalancer(ctx context.Context, service *v1.Service) error { // unless there's an annotation, we can never get a past and current NB to differ, // because they're looked up the same way - if _, ok := service.GetAnnotations()[annLinodeNodeBalancerID]; !ok { + if _, ok := service.GetAnnotations()[annotations.AnnLinodeNodeBalancerID]; !ok { return nil } @@ -201,7 +191,7 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri nb, err = l.getNodeBalancerForService(ctx, service) switch err.(type) { case lbNotFoundError: - if service.GetAnnotations()[annLinodeNodeBalancerID] != "" { + if service.GetAnnotations()[annotations.AnnLinodeNodeBalancerID] != "" { // a load balancer annotation has been created so a NodeBalancer is coming, error out and retry later klog.Infof("NodeBalancer created but not available yet, waiting...") sentry.CaptureError(ctx, err) @@ -238,276 +228,14 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri return lbStatus, nil } -// getNodeBalancerDeviceID gets the deviceID of the nodeBalancer that is attached to the firewall. -func (l *loadbalancers) getNodeBalancerDeviceID(ctx context.Context, firewallID, nbID int) (int, bool, error) { - devices, err := l.client.ListFirewallDevices(ctx, firewallID, &linodego.ListOptions{}) - if err != nil { - return 0, false, err - } - - if len(devices) == 0 { - return 0, false, nil - } - - for _, device := range devices { - if device.Entity.ID == nbID { - return device.ID, true, nil - } - } - - return 0, false, nil -} - -// Updates a service that has a firewallID annotation set. -// If an annotation is set, and the nodebalancer has a firewall that matches the ID, nothing to do -// If there's more than one firewall attached to the node-balancer, an error is returned as its not a supported use case. -// If there's only one firewall attached and it doesn't match what's in the annotation, the new firewall is attached and the old one removed -func (l *loadbalancers) updateFirewallwithID(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer) error { - var newFirewallID int - var err error - - fwID := service.GetAnnotations()[annLinodeCloudFirewallID] - newFirewallID, err = strconv.Atoi(fwID) - if err != nil { - return err - } - - // See if a firewall is attached to the nodebalancer first. - firewalls, err := l.client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) - if err != nil { - return err - } - if len(firewalls) > 1 { - klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) - return errTooManyFirewalls - } - - // get the ID of the firewall that is already attached to the nodeBalancer, if we have one. - var existingFirewallID int - if len(firewalls) == 1 { - existingFirewallID = firewalls[0].ID - } - - // if existing firewall and new firewall differs, attach the new firewall and remove the old. - if existingFirewallID != newFirewallID { - // attach new firewall. - _, err = l.client.CreateFirewallDevice(ctx, newFirewallID, linodego.FirewallDeviceCreateOptions{ - ID: nb.ID, - Type: "nodebalancer", - }) - if err != nil { - return err - } - // remove the existing firewall if it exists - if existingFirewallID != 0 { - deviceID, deviceExists, err := l.getNodeBalancerDeviceID(ctx, existingFirewallID, nb.ID) - if err != nil { - return err - } - - if !deviceExists { - return fmt.Errorf("Error in fetching attached nodeBalancer device") - } - - err = l.client.DeleteFirewallDevice(ctx, existingFirewallID, deviceID) - if err != nil { - return err - } - } - } - return nil -} - -func ipsChanged(ips *linodego.NetworkAddresses, rules []linodego.FirewallRule) bool { - var ruleIPv4s []string - var ruleIPv6s []string - - for _, rule := range rules { - if rule.Addresses.IPv4 != nil { - ruleIPv4s = append(ruleIPv4s, *rule.Addresses.IPv4...) - } - if rule.Addresses.IPv6 != nil { - ruleIPv6s = append(ruleIPv6s, *rule.Addresses.IPv6...) - } - } - - if len(ruleIPv4s) > 0 && ips.IPv4 == nil { - return true - } - - if len(ruleIPv6s) > 0 && ips.IPv6 == nil { - return true - } - - if ips.IPv4 != nil { - for _, ipv4 := range *ips.IPv4 { - if !slices.Contains(ruleIPv4s, ipv4) { - return true - } - } - } - - if ips.IPv6 != nil { - for _, ipv6 := range *ips.IPv6 { - if !slices.Contains(ruleIPv6s, ipv6) { - return true - } - } - } - - return false -} - -func firewallRuleChanged(old linodego.FirewallRuleSet, newACL aclConfig) bool { - var ips *linodego.NetworkAddresses - if newACL.AllowList != nil { - // this is a allowList, this means that the rules should have `DROP` as inboundpolicy - if old.InboundPolicy != "DROP" { - return true - } - if (newACL.AllowList.IPv4 != nil || newACL.AllowList.IPv6 != nil) && len(old.Inbound) == 0 { - return true - } - ips = newACL.AllowList - } - - if newACL.DenyList != nil { - if old.InboundPolicy != "ACCEPT" { - return true - } - - if (newACL.DenyList.IPv4 != nil || newACL.DenyList.IPv6 != nil) && len(old.Inbound) == 0 { - return true - } - ips = newACL.DenyList - } - - return ipsChanged(ips, old.Inbound) -} - -func (l *loadbalancers) updateFWwithACL(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer) error { - // See if a firewall is attached to the nodebalancer first. - firewalls, err := l.client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) - if err != nil { - return err - } - - switch len(firewalls) { - case 0: - { - // need to create a fw and attach it to our nb - fwcreateOpts, err := l.createFirewallOptsForSvc(l.GetLoadBalancerName(ctx, "", service), l.getLoadBalancerTags(ctx, "", service), service) - if err != nil { - return err - } - - fw, err := l.client.CreateFirewall(ctx, *fwcreateOpts) - if err != nil { - return err - } - // attach new firewall. - _, err = l.client.CreateFirewallDevice(ctx, fw.ID, linodego.FirewallDeviceCreateOptions{ - ID: nb.ID, - Type: "nodebalancer", - }) - if err != nil { - return err - } - } - case 1: - { - // We do not want to get into the complexity of reconciling differences, might as well just pull what's in the svc annotation now and update the fw. - var acl aclConfig - err := json.Unmarshal([]byte(service.GetAnnotations()[annLinodeCloudFirewallACL]), &acl) - if err != nil { - return err - } - - changed := firewallRuleChanged(firewalls[0].Rules, acl) - if !changed { - return nil - } - - fwCreateOpts, err := l.createFirewallOptsForSvc(service.Name, []string{""}, service) - if err != nil { - return err - } - _, err = l.client.UpdateFirewallRules(ctx, firewalls[0].ID, fwCreateOpts.Rules) - if err != nil { - return err - } - } - default: - klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) - return errTooManyFirewalls - } - return nil -} - -// updateNodeBalancerFirewall reconciles the firewall attached to the nodebalancer -// -// This function does the following -// 1. If a firewallID annotation is present, it checks if the nodebalancer has a firewall attached, and if it matches the annotationID -// a. If the IDs match, nothing to do here. -// b. If they don't match, the nb is attached to the new firewall and removed from the old one. -// 2. If a firewallACL annotation is present, -// a. it checks if the nodebalancer has a firewall attached, if a fw exists, it updates rules -// b. if a fw does not exist, it creates one -// 3. If neither of these annotations are present, -// a. AND if no firewalls are attached to the nodebalancer, nothing to do. -// b. if the NB has ONE firewall attached, remove it from nb, and clean up if nothing else is attached to it -// c. If there are more than one fw attached to it, then its a problem, return an err -// 4. If both these annotations are present, the firewallID takes precedence, and the ACL annotation is ignored. -// IF a user creates a fw ID externally, and then switches to using a ACL, the CCM will take over the fw that's attached to the nodebalancer. - -func (l *loadbalancers) updateNodeBalancerFirewall(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer) error { - // get the new firewall id from the annotation (if any). - _, fwIDExists := service.GetAnnotations()[annLinodeCloudFirewallID] - if fwIDExists { // If an ID exists, we ignore everything else and handle just that - return l.updateFirewallwithID(ctx, service, nb) - } - - // See if a acl exists - _, fwACLExists := service.GetAnnotations()[annLinodeCloudFirewallACL] - if fwACLExists { // if an ACL exists, but no ID, just update the ACL on the fw. - return l.updateFWwithACL(ctx, service, nb) - } - - // No firewall ID or ACL annotation, see if there are firewalls attached to our nb - firewalls, err := l.client.ListNodeBalancerFirewalls(ctx, nb.ID, &linodego.ListOptions{}) - if err != nil { - return err - } - - if len(firewalls) == 0 { - return nil - } - if len(firewalls) > 1 { - klog.Errorf("Found more than one firewall attached to nodebalancer: %d, firewall IDs: %v", nb.ID, firewalls) - return errTooManyFirewalls - } - - err = l.client.DeleteFirewallDevice(ctx, firewalls[0].ID, nb.ID) - if err != nil { - return err - } - // once we delete the device, we should see if there's anything attached to that firewall - devices, err := l.client.ListFirewallDevices(ctx, firewalls[0].ID, &linodego.ListOptions{}) - if err != nil { - return err - } - - if len(devices) == 0 { - // nothing attached to it, clean it up - return l.client.DeleteFirewall(ctx, firewalls[0].ID) - } - // else let that firewall linger, don't mess with it. - - return nil -} - //nolint:funlen -func (l *loadbalancers) updateNodeBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, nb *linodego.NodeBalancer) (err error) { +func (l *loadbalancers) updateNodeBalancer( + ctx context.Context, + clusterName string, + service *v1.Service, + nodes []*v1.Node, + nb *linodego.NodeBalancer, +) (err error) { if len(nodes) == 0 { return fmt.Errorf("%w: service %s", errNoNodesAvailable, getServiceNn(service)) } @@ -523,7 +251,7 @@ func (l *loadbalancers) updateNodeBalancer(ctx context.Context, clusterName stri } } - tags := l.getLoadBalancerTags(ctx, clusterName, service) + tags := l.GetLoadBalancerTags(ctx, clusterName, service) if !reflect.DeepEqual(nb.Tags, tags) { update := nb.GetUpdateOptions() update.Tags = &tags @@ -534,7 +262,8 @@ func (l *loadbalancers) updateNodeBalancer(ctx context.Context, clusterName stri } } - err = l.updateNodeBalancerFirewall(ctx, service, nb) + fwClient := firewall.LinodeClient{Client: l.client} + err = fwClient.UpdateNodeBalancerFirewall(ctx, l.GetLoadBalancerName(ctx, clusterName, service), tags, service, nb) if err != nil { return err } @@ -666,7 +395,7 @@ func (l *loadbalancers) deleteUnusedConfigs(ctx context.Context, nbConfigs []lin // shouldPreserveNodeBalancer determines whether a NodeBalancer should be deleted based on the // service's preserve annotation. func (l *loadbalancers) shouldPreserveNodeBalancer(service *v1.Service) bool { - return getServiceBoolAnnotation(service, annLinodeLoadBalancerPreserve) + return getServiceBoolAnnotation(service, annotations.AnnLinodeLoadBalancerPreserve) } // EnsureLoadBalancerDeleted deletes the specified loadbalancer if it exists. @@ -702,7 +431,12 @@ func (l *loadbalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa } if l.shouldPreserveNodeBalancer(service) { - klog.Infof("short-circuiting deletion of NodeBalancer (%d) for service (%s) as annotated with %s", nb.ID, serviceNn, annLinodeLoadBalancerPreserve) + klog.Infof( + "short-circuiting deletion of NodeBalancer (%d) for service (%s) as annotated with %s", + nb.ID, + serviceNn, + annotations.AnnLinodeLoadBalancerPreserve, + ) return nil } @@ -754,13 +488,13 @@ func (l *loadbalancers) getNodeBalancerByID(ctx context.Context, service *v1.Ser return nb, nil } -func (l *loadbalancers) getLoadBalancerTags(_ context.Context, clusterName string, service *v1.Service) []string { +func (l *loadbalancers) GetLoadBalancerTags(_ context.Context, clusterName string, service *v1.Service) []string { tags := []string{} if clusterName != "" { tags = append(tags, clusterName) } - tagStr, ok := service.GetAnnotations()[annLinodeLoadBalancerTags] + tagStr, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerTags] if ok { return append(tags, strings.Split(tagStr, ",")...) } @@ -768,149 +502,11 @@ func (l *loadbalancers) getLoadBalancerTags(_ context.Context, clusterName strin return tags } -func chunkIPs(ips []string) [][]string { - chunks := [][]string{} - ipCount := len(ips) - - // If the number of IPs is less than or equal to maxIPsPerFirewall, - // return a single chunk containing all IPs. - if ipCount <= maxIPsPerFirewall { - return [][]string{ips} - } - - // Otherwise, break the IPs into chunks with maxIPsPerFirewall IPs per chunk. - chunkCount := 0 - for ipCount > maxIPsPerFirewall { - start := chunkCount * maxIPsPerFirewall - end := (chunkCount + 1) * maxIPsPerFirewall - chunks = append(chunks, ips[start:end]) - chunkCount++ - ipCount -= maxIPsPerFirewall - } - - // Append the remaining IPs as a chunk. - chunks = append(chunks, ips[chunkCount*maxIPsPerFirewall:]) - - return chunks -} - -// processACL takes the IPs, aclType, label etc and formats them into the passed linodego.FirewallCreateOptions pointer. -func processACL(fwcreateOpts *linodego.FirewallCreateOptions, aclType, label, svcName, ports string, ips linodego.NetworkAddresses) error { - ruleLabel := fmt.Sprintf("%s-%s", aclType, svcName) - if len(ruleLabel) > maxFirewallRuleLabelLen { - newLabel := ruleLabel[0:maxFirewallRuleLabelLen] - klog.Infof("Firewall label '%s' is too long. Stripping to '%s'", ruleLabel, newLabel) - ruleLabel = newLabel - } - - // Linode has a limitation of firewall rules with a max of 255 IPs per rule - var ipv4s, ipv6s []string // doing this to avoid dereferencing a nil pointer - if ips.IPv6 != nil { - ipv6s = *ips.IPv6 - } - if ips.IPv4 != nil { - ipv4s = *ips.IPv4 - } - - if len(ipv4s)+len(ipv6s) > maxIPsPerFirewall { - ipv4chunks := chunkIPs(ipv4s) - for i, chunk := range ipv4chunks { - v4chunk := chunk - fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ - Action: aclType, - Label: ruleLabel, - Description: fmt.Sprintf("Rule %d, Created by linode-ccm: %s, for %s", i, label, svcName), - Protocol: linodego.TCP, // Nodebalancers support only TCP. - Ports: ports, - Addresses: linodego.NetworkAddresses{IPv4: &v4chunk}, - }) - } - - ipv6chunks := chunkIPs(ipv6s) - for i, chunk := range ipv6chunks { - v6chunk := chunk - fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ - Action: aclType, - Label: ruleLabel, - Description: fmt.Sprintf("Rule %d, Created by linode-ccm: %s, for %s", i, label, svcName), - Protocol: linodego.TCP, // Nodebalancers support only TCP. - Ports: ports, - Addresses: linodego.NetworkAddresses{IPv6: &v6chunk}, - }) - } - } else { - fwcreateOpts.Rules.Inbound = append(fwcreateOpts.Rules.Inbound, linodego.FirewallRule{ - Action: aclType, - Label: ruleLabel, - Description: fmt.Sprintf("Created by linode-ccm: %s, for %s", label, svcName), - Protocol: linodego.TCP, // Nodebalancers support only TCP. - Ports: ports, - Addresses: ips, - }) - } - - fwcreateOpts.Rules.OutboundPolicy = "ACCEPT" - if aclType == "ACCEPT" { - // if an allowlist is present, we drop everything else. - fwcreateOpts.Rules.InboundPolicy = "DROP" - } else { - // if a denylist is present, we accept everything else. - fwcreateOpts.Rules.InboundPolicy = "ACCEPT" - } - - if len(fwcreateOpts.Rules.Inbound) > maxRulesPerFirewall { - return errTooManyIPs - } - return nil -} - -type aclConfig struct { - AllowList *linodego.NetworkAddresses `json:"allowList"` - DenyList *linodego.NetworkAddresses `json:"denyList"` -} - -func (l *loadbalancers) createFirewallOptsForSvc(label string, tags []string, svc *v1.Service) (*linodego.FirewallCreateOptions, error) { - // Fetch acl from annotation - aclString := svc.GetAnnotations()[annLinodeCloudFirewallACL] - fwcreateOpts := linodego.FirewallCreateOptions{ - Label: label, - Tags: tags, - } - servicePorts := make([]string, 0, len(svc.Spec.Ports)) - for _, port := range svc.Spec.Ports { - servicePorts = append(servicePorts, strconv.Itoa(int(port.Port))) - } - - portsString := strings.Join(servicePorts[:], ",") - var acl aclConfig - err := json.Unmarshal([]byte(aclString), &acl) - if err != nil { - return nil, err - } - // it is a problem if both are set, or if both are not set - if (acl.AllowList != nil && acl.DenyList != nil) || (acl.AllowList == nil && acl.DenyList == nil) { - return nil, errInvalidFWConfig - } - - aclType := "ACCEPT" - allowedIPs := acl.AllowList - if acl.DenyList != nil { - aclType = "DROP" - allowedIPs = acl.DenyList - } - - err = processACL(&fwcreateOpts, aclType, label, svc.Name, portsString, *allowedIPs) - if err != nil { - return nil, err - } - return &fwcreateOpts, nil -} - func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName string, service *v1.Service, configs []*linodego.NodeBalancerConfigCreateOptions) (lb *linodego.NodeBalancer, err error) { connThrottle := getConnectionThrottle(service) label := l.GetLoadBalancerName(ctx, clusterName, service) - tags := l.getLoadBalancerTags(ctx, clusterName, service) + tags := l.GetLoadBalancerTags(ctx, clusterName, service) createOpts := linodego.NodeBalancerCreateOptions{ Label: &label, Region: l.zone, @@ -919,7 +515,7 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri Tags: tags, } - fwid, ok := service.GetAnnotations()[annLinodeCloudFirewallID] + fwid, ok := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallID] if ok { firewallID, err := strconv.Atoi(fwid) if err != nil { @@ -928,18 +524,18 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri createOpts.FirewallID = firewallID } else { // There's no firewallID already set, see if we need to create a new fw, look for the acl annotation. - _, ok := service.GetAnnotations()[annLinodeCloudFirewallACL] + _, ok := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallACL] if ok { - fwcreateOpts, err := l.createFirewallOptsForSvc(label, tags, service) + fwcreateOpts, err := firewall.CreateFirewallOptsForSvc(label, tags, service) if err != nil { return nil, err } - firewall, err := l.client.CreateFirewall(ctx, *fwcreateOpts) + fw, err := l.client.CreateFirewall(ctx, *fwcreateOpts) if err != nil { return nil, err } - createOpts.FirewallID = firewall.ID + createOpts.FirewallID = fw.ID } // no need to deal with firewalls, continue creating nb's } @@ -947,14 +543,6 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri return l.client.CreateNodeBalancer(ctx, createOpts) } -func (l *loadbalancers) createFirewall(ctx context.Context, opts linodego.FirewallCreateOptions) (fw *linodego.Firewall, err error) { - return l.client.CreateFirewall(ctx, opts) -} - -func (l *loadbalancers) deleteFirewall(ctx context.Context, firewall *linodego.Firewall) error { - return l.client.DeleteFirewall(ctx, firewall.ID) -} - //nolint:funlen func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1.Service, port int) (linodego.NodeBalancerConfig, error) { portConfig, err := getPortConfig(service, port) @@ -975,7 +563,7 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1 } if health == linodego.CheckHTTP || health == linodego.CheckHTTPBody { - path := service.GetAnnotations()[annLinodeCheckPath] + path := service.GetAnnotations()[annotations.AnnLinodeCheckPath] if path == "" { path = "/" } @@ -983,14 +571,14 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1 } if health == linodego.CheckHTTPBody { - body := service.GetAnnotations()[annLinodeCheckBody] + body := service.GetAnnotations()[annotations.AnnLinodeCheckBody] if body == "" { - return config, fmt.Errorf("for health check type http_body need body regex annotation %v", annLinodeCheckBody) + return config, fmt.Errorf("for health check type http_body need body regex annotation %v", annotations.AnnLinodeCheckBody) } config.CheckBody = body } checkInterval := 5 - if ci, ok := service.GetAnnotations()[annLinodeHealthCheckInterval]; ok { + if ci, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckInterval]; ok { if checkInterval, err = strconv.Atoi(ci); err != nil { return config, err } @@ -998,7 +586,7 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1 config.CheckInterval = checkInterval checkTimeout := 3 - if ct, ok := service.GetAnnotations()[annLinodeHealthCheckTimeout]; ok { + if ct, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckTimeout]; ok { if checkTimeout, err = strconv.Atoi(ct); err != nil { return config, err } @@ -1006,7 +594,7 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1 config.CheckTimeout = checkTimeout checkAttempts := 2 - if ca, ok := service.GetAnnotations()[annLinodeHealthCheckAttempts]; ok { + if ca, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckAttempts]; ok { if checkAttempts, err = strconv.Atoi(ca); err != nil { return config, err } @@ -1014,7 +602,7 @@ func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1 config.CheckAttempts = checkAttempts checkPassive := true - if cp, ok := service.GetAnnotations()[annLinodeHealthCheckPassive]; ok { + if cp, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckPassive]; ok { if checkPassive, err = strconv.ParseBool(cp); err != nil { return config, err } @@ -1135,7 +723,7 @@ func getPortConfig(service *v1.Service, port int) (portConfig, error) { protocol := portConfigAnnotation.Protocol if protocol == "" { protocol = "tcp" - if p, ok := service.GetAnnotations()[annLinodeDefaultProtocol]; ok { + if p, ok := service.GetAnnotations()[annotations.AnnLinodeDefaultProtocol]; ok { protocol = p } } @@ -1144,7 +732,7 @@ func getPortConfig(service *v1.Service, port int) (portConfig, error) { proxyProtocol := portConfigAnnotation.ProxyProtocol if proxyProtocol == "" { proxyProtocol = string(linodego.ProxyProtocolNone) - for _, ann := range []string{annLinodeDefaultProxyProtocol, annLinodeProxyProtocolDeprecated} { + for _, ann := range []string{annotations.AnnLinodeDefaultProxyProtocol, annLinodeProxyProtocolDeprecated} { if pp, ok := service.GetAnnotations()[ann]; ok { proxyProtocol = pp break @@ -1172,19 +760,19 @@ func getPortConfig(service *v1.Service, port int) (portConfig, error) { } func getHealthCheckType(service *v1.Service) (linodego.ConfigCheck, error) { - hType, ok := service.GetAnnotations()[annLinodeHealthCheckType] + hType, ok := service.GetAnnotations()[annotations.AnnLinodeHealthCheckType] if !ok { return linodego.CheckConnection, nil } if hType != "none" && hType != "connection" && hType != "http" && hType != "http_body" { - return "", fmt.Errorf("invalid health check type: %q specified in annotation: %q", hType, annLinodeHealthCheckType) + return "", fmt.Errorf("invalid health check type: %q specified in annotation: %q", hType, annotations.AnnLinodeHealthCheckType) } return linodego.ConfigCheck(hType), nil } func getPortConfigAnnotation(service *v1.Service, port int) (portConfigAnnotation, error) { annotation := portConfigAnnotation{} - annotationKey := annLinodePortConfigPrefix + strconv.Itoa(port) + annotationKey := annotations.AnnLinodePortConfigPrefix + strconv.Itoa(port) annotationJSON, ok := service.GetAnnotations()[annotationKey] if !ok { @@ -1204,7 +792,7 @@ func getPortConfigAnnotation(service *v1.Service, port int) (portConfigAnnotatio // network, this will not be the NodeInternalIP, so this prefers an annotation // cluster operators may specify in such a situation. func getNodePrivateIP(node *v1.Node) string { - if address, exists := node.Annotations[annLinodeNodePrivateIP]; exists { + if address, exists := node.Annotations[annotations.AnnLinodeNodePrivateIP]; exists { return address } @@ -1239,7 +827,7 @@ func getTLSCertInfo(ctx context.Context, kubeClient kubernetes.Interface, namesp func getConnectionThrottle(service *v1.Service) int { connThrottle := 20 - if connThrottleString := service.GetAnnotations()[annLinodeThrottle]; connThrottleString != "" { + if connThrottleString := service.GetAnnotations()[annotations.AnnLinodeThrottle]; connThrottleString != "" { parsed, err := strconv.Atoi(connThrottleString) if err == nil { if parsed < 0 { @@ -1260,7 +848,7 @@ func makeLoadBalancerStatus(service *v1.Service, nb *linodego.NodeBalancer) *v1. ingress := v1.LoadBalancerIngress{ Hostname: *nb.Hostname, } - if !getServiceBoolAnnotation(service, annLinodeHostnameOnlyIngress) { + if !getServiceBoolAnnotation(service, annotations.AnnLinodeHostnameOnlyIngress) { if val := envBoolOptions("LINODE_HOSTNAME_ONLY_INGRESS"); val { klog.Infof("LINODE_HOSTNAME_ONLY_INGRESS: (%v)", val) } else { diff --git a/cloud/linode/loadbalancers_test.go b/cloud/linode/loadbalancers_test.go index 1a44bd00..f944bc40 100644 --- a/cloud/linode/loadbalancers_test.go +++ b/cloud/linode/loadbalancers_test.go @@ -24,6 +24,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/firewall" ) const testCert string = `-----BEGIN CERTIFICATE----- @@ -262,14 +265,14 @@ func stubService(fake *fake.Clientset, service *v1.Service) { _, _ = fake.CoreV1().Services("").Create(context.TODO(), service, metav1.CreateOptions{}) } -func testCreateNodeBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI, annotations map[string]string) error { +func testCreateNodeBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI, annMap map[string]string) error { svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeThrottle: "15", - annLinodeLoadBalancerTags: "fake,test,yolo", + annotations.AnnLinodeThrottle: "15", + annotations.AnnLinodeLoadBalancerTags: "fake,test,yolo", }, }, Spec: v1.ServiceSpec{ @@ -289,7 +292,7 @@ func testCreateNodeBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI, a }, }, } - for key, value := range annotations { + for key, value := range annMap { svc.Annotations[key] = value } lb := &loadbalancers{client, "us-west", nil} @@ -336,7 +339,7 @@ func testCreateNodeBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI, a t.Logf("actual: %v", nb.Tags) } - _, ok := annotations[annLinodeCloudFirewallACL] + _, ok := annMap[annotations.AnnLinodeCloudFirewallACL] if ok { // a firewall was configured for this firewalls, err := client.ListNodeBalancerFirewalls(context.TODO(), nb.ID, &linodego.ListOptions{}) @@ -362,18 +365,18 @@ func testCreateNodeBalancerWithOutFirewall(t *testing.T, client *linodego.Client func testCreateNodeBalanceWithNoAllowOrDenyList(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallACL: `{}`, + annotations.AnnLinodeCloudFirewallACL: `{}`, } err := testCreateNodeBalancer(t, client, f, annotations) - if err == nil || !stderrors.Is(err, errInvalidFWConfig) { - t.Fatalf("expected a %v error, got %v", errInvalidFWConfig, err) + if err == nil || !stderrors.Is(err, firewall.ErrInvalidFWConfig) { + t.Fatalf("expected a %v error, got %v", firewall.ErrInvalidFWConfig, err) } } func testCreateNodeBalanceWithBothAllowOrDenyList(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"] }, @@ -384,14 +387,14 @@ func testCreateNodeBalanceWithBothAllowOrDenyList(t *testing.T, client *linodego } err := testCreateNodeBalancer(t, client, f, annotations) - if err == nil || !stderrors.Is(err, errInvalidFWConfig) { - t.Fatalf("expected a %v error, got %v", errInvalidFWConfig, err) + if err == nil || !stderrors.Is(err, firewall.ErrInvalidFWConfig) { + t.Fatalf("expected a %v error, got %v", firewall.ErrInvalidFWConfig, err) } } func testCreateNodeBalancerWithAllowList(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"] } @@ -406,7 +409,7 @@ func testCreateNodeBalancerWithAllowList(t *testing.T, client *linodego.Client, func testCreateNodeBalancerWithDenyList(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "denyList": { "ipv4": ["2.2.2.2"] } @@ -421,7 +424,7 @@ func testCreateNodeBalancerWithDenyList(t *testing.T, client *linodego.Client, f func testCreateNodeBalancerWithFirewall(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallID: "123", + annotations.AnnLinodeCloudFirewallID: "123", } err := testCreateNodeBalancer(t, client, f, annotations) if err != nil { @@ -431,7 +434,7 @@ func testCreateNodeBalancerWithFirewall(t *testing.T, client *linodego.Client, f func testCreateNodeBalancerWithInvalidFirewall(t *testing.T, client *linodego.Client, f *fakeAPI) { annotations := map[string]string{ - annLinodeCloudFirewallID: "qwerty", + annotations.AnnLinodeCloudFirewallID: "qwerty", } expectedError := "strconv.Atoi: parsing \"qwerty\": invalid syntax" err := testCreateNodeBalancer(t, client, f, annotations) @@ -446,7 +449,7 @@ func testUpdateLoadBalancerAddAnnotation(t *testing.T, client *linodego.Client, Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeThrottle: "15", + annotations.AnnLinodeThrottle: "15", }, }, Spec: v1.ServiceSpec{ @@ -490,7 +493,7 @@ func testUpdateLoadBalancerAddAnnotation(t *testing.T, client *linodego.Client, stubService(fakeClientset, svc) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeThrottle: "10", + annotations.AnnLinodeThrottle: "10", }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -512,7 +515,7 @@ func testUpdateLoadBalancerAddAnnotation(t *testing.T, client *linodego.Client, func testUpdateLoadBalancerAddPortAnnotation(t *testing.T, client *linodego.Client, _ *fakeAPI) { targetTestPort := 80 - portConfigAnnotation := fmt.Sprintf("%s%d", annLinodePortConfigPrefix, targetTestPort) + portConfigAnnotation := fmt.Sprintf("%s%d", annotations.AnnLinodePortConfigPrefix, targetTestPort) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: randString(), @@ -642,7 +645,7 @@ func testUpdateLoadBalancerAddTags(t *testing.T, client *linodego.Client, _ *fak testTags := "test,new,tags" svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeLoadBalancerTags: testTags, + annotations.AnnLinodeLoadBalancerTags: testTags, }) err = lb.UpdateLoadBalancer(context.TODO(), clusterName, svc, nodes) @@ -669,7 +672,7 @@ func testUpdateLoadBalancerAddTLSPort(t *testing.T, client *linodego.Client, _ * Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeThrottle: "15", + annotations.AnnLinodeThrottle: "15", }, }, Spec: v1.ServiceSpec{ @@ -723,7 +726,7 @@ func testUpdateLoadBalancerAddTLSPort(t *testing.T, client *linodego.Client, _ * stubService(fakeClientset, svc) svc.Spec.Ports = append(svc.Spec.Ports, extraPort) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodePortConfigPrefix + "443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, + annotations.AnnLinodePortConfigPrefix + "443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) if err != nil { @@ -837,7 +840,7 @@ func testUpdateLoadBalancerAddProxyProtocol(t *testing.T, client *linodego.Clien svc.Status.LoadBalancer = *makeLoadBalancerStatus(svc, nodeBalancer) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeDefaultProxyProtocol: string(tc.proxyProtocolConfig), + annotations.AnnLinodeDefaultProxyProtocol: string(tc.proxyProtocolConfig), }) stubService(fakeClientset, svc) @@ -874,7 +877,7 @@ func testUpdateLoadBalancerAddNewFirewall(t *testing.T, client *linodego.Client, Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeThrottle: "15", + annotations.AnnLinodeThrottle: "15", }, }, Spec: v1.ServiceSpec{ @@ -916,7 +919,8 @@ func testUpdateLoadBalancerAddNewFirewall(t *testing.T, client *linodego.Client, } svc.Status.LoadBalancer = *lbStatus stubService(fakeClientset, svc) - firewall, err := lb.createFirewall(context.TODO(), linodego.FirewallCreateOptions{ + fwClient := firewall.LinodeClient{Client: client} + fw, err := fwClient.CreateFirewall(context.TODO(), linodego.FirewallCreateOptions{ Label: "test", Rules: linodego.FirewallRuleSet{Inbound: []linodego.FirewallRule{{ Action: "ACCEPT", @@ -933,11 +937,11 @@ func testUpdateLoadBalancerAddNewFirewall(t *testing.T, client *linodego.Client, t.Errorf("CreatingFirewall returned an error: %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewall) + _ = fwClient.DeleteFirewall(context.TODO(), fw) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewall.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(fw.ID), }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -959,7 +963,7 @@ func testUpdateLoadBalancerAddNewFirewall(t *testing.T, client *linodego.Client, t.Fatalf("No attached firewalls found") } - if firewalls[0].ID != firewall.ID { + if firewalls[0].ID != fw.ID { t.Fatalf("Attached firewallID not matching with created firewall") } } @@ -1060,7 +1064,7 @@ func testUpdateLoadBalancerAddNewFirewallACL(t *testing.T, client *linodego.Clie } svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallACL: string(aclString), + annotations.AnnLinodeCloudFirewallACL: string(aclString), }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -1085,7 +1089,7 @@ func testUpdateLoadBalancerAddNewFirewallACL(t *testing.T, client *linodego.Clie if firewallsNew[0].Rules.InboundPolicy != "DROP" { t.Errorf("expected DROP inbound policy, got %s", firewallsNew[0].Rules.InboundPolicy) } - + if len(firewallsNew[0].Rules.Inbound) != 4 { t.Errorf("expected 4 rules, got %d", len(firewallsNew[0].Rules.Inbound)) } @@ -1127,7 +1131,7 @@ func testUpdateLoadBalancerUpdateFirewallRemoveACLaddID(t *testing.T, client *li lb.kubeClient = fakeClientset svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"] } @@ -1167,7 +1171,8 @@ func testUpdateLoadBalancerUpdateFirewallRemoveACLaddID(t *testing.T, client *li t.Errorf("expected IP, got %v", fwIPs) } - firewall, err := lb.createFirewall(context.TODO(), linodego.FirewallCreateOptions{ + fwClient := firewall.LinodeClient{Client: client} + fw, err := fwClient.CreateFirewall(context.TODO(), linodego.FirewallCreateOptions{ Label: "test", Rules: linodego.FirewallRuleSet{Inbound: []linodego.FirewallRule{{ Action: "ACCEPT", @@ -1184,11 +1189,11 @@ func testUpdateLoadBalancerUpdateFirewallRemoveACLaddID(t *testing.T, client *li t.Errorf("Error creating firewall %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewall) + _ = fwClient.DeleteFirewall(context.TODO(), fw) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewall.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(fw.ID), }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -1219,7 +1224,7 @@ func testUpdateLoadBalancerUpdateFirewallRemoveACLaddID(t *testing.T, client *li t.Errorf("expected 2.2.2.2, got %v", fwIPs) } - if firewallsNew[0].ID != firewall.ID { + if firewallsNew[0].ID != fw.ID { t.Errorf("Firewall ID does not match what we created, something wrong.") } } @@ -1259,7 +1264,8 @@ func testUpdateLoadBalancerUpdateFirewallRemoveIDaddACL(t *testing.T, client *li fakeClientset := fake.NewSimpleClientset() lb.kubeClient = fakeClientset - firewall, err := lb.createFirewall(context.TODO(), linodego.FirewallCreateOptions{ + fwClient := firewall.LinodeClient{Client: client} + fw, err := fwClient.CreateFirewall(context.TODO(), linodego.FirewallCreateOptions{ Label: "test", Rules: linodego.FirewallRuleSet{Inbound: []linodego.FirewallRule{{ Action: "ACCEPT", @@ -1276,11 +1282,11 @@ func testUpdateLoadBalancerUpdateFirewallRemoveIDaddACL(t *testing.T, client *li t.Errorf("Error creating firewall %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewall) + _ = fwClient.DeleteFirewall(context.TODO(), fw) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewall.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(fw.ID), }) defer func() { @@ -1316,7 +1322,7 @@ func testUpdateLoadBalancerUpdateFirewallRemoveIDaddACL(t *testing.T, client *li t.Errorf("expected IP, got %v", fwIPs) } svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"] } @@ -1351,7 +1357,7 @@ func testUpdateLoadBalancerUpdateFirewallRemoveIDaddACL(t *testing.T, client *li t.Errorf("expected 2.2.2.2, got %v", fwIPs) } - if firewallsNew[0].ID != firewall.ID { + if firewallsNew[0].ID != fw.ID { t.Errorf("Firewall ID does not match, something wrong.") } } @@ -1362,7 +1368,7 @@ func testUpdateLoadBalancerUpdateFirewallACL(t *testing.T, client *linodego.Clie Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"] } @@ -1434,7 +1440,7 @@ func testUpdateLoadBalancerUpdateFirewallACL(t *testing.T, client *linodego.Clie fmt.Printf("got %v", fwIPs) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallACL: `{ + annotations.AnnLinodeCloudFirewallACL: `{ "allowList": { "ipv4": ["2.2.2.2"], "ipv6": ["dead:beef::/128"] @@ -1499,7 +1505,7 @@ func testUpdateLoadBalancerUpdateFirewall(t *testing.T, client *linodego.Client, Name: randString(), UID: "foobar123", Annotations: map[string]string{ - annLinodeThrottle: "15", + annotations.AnnLinodeThrottle: "15", }, }, Spec: v1.ServiceSpec{ @@ -1535,16 +1541,17 @@ func testUpdateLoadBalancerUpdateFirewall(t *testing.T, client *linodego.Client, _ = lb.EnsureLoadBalancerDeleted(context.TODO(), "linodelb", svc) }() - firewall, err := lb.createFirewall(context.TODO(), firewallCreateOpts) + fwClient := firewall.LinodeClient{Client: client} + fw, err := fwClient.CreateFirewall(context.TODO(), firewallCreateOpts) if err != nil { t.Errorf("Error creating firewall %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewall) + _ = fwClient.DeleteFirewall(context.TODO(), fw) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewall.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(fw.ID), }) lbStatus, err := lb.EnsureLoadBalancer(context.TODO(), "linodelb", svc, nodes) if err != nil { @@ -1567,21 +1574,21 @@ func testUpdateLoadBalancerUpdateFirewall(t *testing.T, client *linodego.Client, t.Fatalf("No firewalls attached") } - if firewall.ID != firewalls[0].ID { + if fw.ID != firewalls[0].ID { t.Fatalf("Attached firewallID not matching with created firewall") } firewallCreateOpts.Label = "test2" - firewallNew, err := lb.createFirewall(context.TODO(), firewallCreateOpts) + firewallNew, err := fwClient.CreateFirewall(context.TODO(), firewallCreateOpts) if err != nil { t.Fatalf("Error in creating firewall %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewallNew) + _ = fwClient.DeleteFirewall(context.TODO(), firewallNew) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewallNew.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(firewallNew.ID), }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -1661,16 +1668,17 @@ func testUpdateLoadBalancerDeleteFirewall(t *testing.T, client *linodego.Client, _ = lb.EnsureLoadBalancerDeleted(context.TODO(), "linodelb", svc) }() - firewall, err := lb.createFirewall(context.TODO(), firewallCreateOpts) + fwClient := firewall.LinodeClient{Client: client} + fw, err := fwClient.CreateFirewall(context.TODO(), firewallCreateOpts) if err != nil { t.Errorf("Error in creating firewall %s", err) } defer func() { - _ = lb.deleteFirewall(context.TODO(), firewall) + _ = fwClient.DeleteFirewall(context.TODO(), fw) }() svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeCloudFirewallID: strconv.Itoa(firewall.ID), + annotations.AnnLinodeCloudFirewallID: strconv.Itoa(fw.ID), }) lbStatus, err := lb.EnsureLoadBalancer(context.TODO(), "linodelb", svc, nodes) @@ -1694,7 +1702,7 @@ func testUpdateLoadBalancerDeleteFirewall(t *testing.T, client *linodego.Client, t.Fatalf("No firewalls attached") } - if firewall.ID != firewalls[0].ID { + if fw.ID != firewalls[0].ID { t.Fatalf("Attached firewallID not matching with created firewall") } @@ -1773,7 +1781,7 @@ func testUpdateLoadBalancerAddNodeBalancerID(t *testing.T, client *linodego.Clie stubService(fakeClientset, svc) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeNodeBalancerID: strconv.Itoa(newNodeBalancer.ID), + annotations.AnnLinodeNodeBalancerID: strconv.Itoa(newNodeBalancer.ID), }) err = lb.UpdateLoadBalancer(context.TODO(), "linodelb", svc, nodes) if err != nil { @@ -1819,7 +1827,7 @@ func Test_getConnectionThrottle(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeThrottle: "foo", + annotations.AnnLinodeThrottle: "foo", }, }, }, @@ -1832,7 +1840,7 @@ func Test_getConnectionThrottle(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeThrottle: "-123", + annotations.AnnLinodeThrottle: "-123", }, }, }, @@ -1845,7 +1853,7 @@ func Test_getConnectionThrottle(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeThrottle: "1", + annotations.AnnLinodeThrottle: "1", }, }, }, @@ -1858,7 +1866,7 @@ func Test_getConnectionThrottle(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeThrottle: "21", + annotations.AnnLinodeThrottle: "21", }, }, }, @@ -1902,7 +1910,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), + annotations.AnnLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), }, }, }, @@ -1916,8 +1924,8 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), - annLinodePortConfigPrefix + "443": fmt.Sprintf(`{"proxy-protocol": "%s"}`, linodego.ProxyProtocolV1), + annotations.AnnLinodeDefaultProxyProtocol: string(linodego.ProxyProtocolV2), + annotations.AnnLinodePortConfigPrefix + "443": fmt.Sprintf(`{"proxy-protocol": "%s"}`, linodego.ProxyProtocolV1), }, }, }, @@ -1931,7 +1939,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProxyProtocol: "invalid", + annotations.AnnLinodeDefaultProxyProtocol: "invalid", }, }, }, @@ -1957,7 +1965,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, }, @@ -1971,7 +1979,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "HTTP", + annotations.AnnLinodeDefaultProtocol: "HTTP", }, }, }, @@ -1985,7 +1993,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "invalid", + annotations.AnnLinodeDefaultProtocol: "invalid", }, }, }, @@ -1999,8 +2007,8 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "http", - annLinodePortConfigPrefix + "443": `{}`, + annotations.AnnLinodeDefaultProtocol: "http", + annotations.AnnLinodePortConfigPrefix + "443": `{}`, }, }, }, @@ -2014,7 +2022,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodePortConfigPrefix + "443": `{ "protocol": "HTTp" }`, + annotations.AnnLinodePortConfigPrefix + "443": `{ "protocol": "HTTp" }`, }, }, }, @@ -2028,7 +2036,7 @@ func Test_getPortConfig(t *testing.T) { Name: randString(), UID: "abc123", Annotations: map[string]string{ - annLinodePortConfigPrefix + "443": `{ "protocol": "invalid" }`, + annotations.AnnLinodePortConfigPrefix + "443": `{ "protocol": "invalid" }`, }, }, }, @@ -2083,7 +2091,7 @@ func Test_getHealthCheckType(t *testing.T) { Name: "test", UID: "abc123", Annotations: map[string]string{ - annLinodeHealthCheckType: "http", + annotations.AnnLinodeHealthCheckType: "http", }, }, }, @@ -2097,12 +2105,12 @@ func Test_getHealthCheckType(t *testing.T) { Name: "test", UID: "abc123", Annotations: map[string]string{ - annLinodeHealthCheckType: "invalid", + annotations.AnnLinodeHealthCheckType: "invalid", }, }, }, "", - fmt.Errorf("invalid health check type: %q specified in annotation: %q", "invalid", annLinodeHealthCheckType), + fmt.Errorf("invalid health check type: %q specified in annotation: %q", "invalid", annotations.AnnLinodeHealthCheckType), }, } @@ -2163,7 +2171,7 @@ func Test_getNodePrivateIP(t *testing.T) { &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - annLinodeNodePrivateIP: "192.168.42.42", + annotations.AnnLinodeNodePrivateIP: "192.168.42.42", }, }, Status: v1.NodeStatus{ @@ -2197,7 +2205,7 @@ func testBuildLoadBalancerRequest(t *testing.T, client *linodego.Client, _ *fake Name: "test", UID: "foobar123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, Spec: v1.ServiceSpec{ @@ -2284,17 +2292,17 @@ func testEnsureLoadBalancerPreserveAnnotation(t *testing.T, client *linodego.Cli }{ { name: "load balancer preserved", - annotations: map[string]string{annLinodeLoadBalancerPreserve: "true"}, + annotations: map[string]string{annotations.AnnLinodeLoadBalancerPreserve: "true"}, deleted: false, }, { name: "load balancer not preserved (deleted)", - annotations: map[string]string{annLinodeLoadBalancerPreserve: "false"}, + annotations: map[string]string{annotations.AnnLinodeLoadBalancerPreserve: "false"}, deleted: true, }, { name: "invalid value treated as false (deleted)", - annotations: map[string]string{annLinodeLoadBalancerPreserve: "bogus"}, + annotations: map[string]string{annotations.AnnLinodeLoadBalancerPreserve: "bogus"}, deleted: true, }, } { @@ -2336,7 +2344,7 @@ func testEnsureLoadBalancerDeleted(t *testing.T, client *linodego.Client, fake * Name: "test", UID: "foobar123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, Spec: v1.ServiceSpec{ @@ -2370,7 +2378,7 @@ func testEnsureLoadBalancerDeleted(t *testing.T, client *linodego.Client, fake * Name: "notexists", UID: "notexists123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, Spec: v1.ServiceSpec{ @@ -2414,8 +2422,8 @@ func testEnsureExistingLoadBalancer(t *testing.T, client *linodego.Client, _ *fa Name: "testensure", UID: "foobar123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", - annLinodePortConfigPrefix + "8443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, + annotations.AnnLinodeDefaultProtocol: "tcp", + annotations.AnnLinodePortConfigPrefix + "8443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, }, }, Spec: v1.ServiceSpec{ @@ -2560,11 +2568,11 @@ func testMakeLoadBalancerStatus(t *testing.T, client *linodego.Client, _ *fakeAP t.Errorf("expected status for basic service to be %#v; got %#v", expectedStatus, status) } - svc.Annotations[annLinodeHostnameOnlyIngress] = "true" + svc.Annotations[annotations.AnnLinodeHostnameOnlyIngress] = "true" expectedStatus.Ingress[0] = v1.LoadBalancerIngress{Hostname: hostname} status = makeLoadBalancerStatus(svc, nb) if !reflect.DeepEqual(status, expectedStatus) { - t.Errorf("expected status for %q annotated service to be %#v; got %#v", annLinodeHostnameOnlyIngress, expectedStatus, status) + t.Errorf("expected status for %q annotated service to be %#v; got %#v", annotations.AnnLinodeHostnameOnlyIngress, expectedStatus, status) } } @@ -2598,21 +2606,21 @@ func testMakeLoadBalancerStatusEnvVar(t *testing.T, client *linodego.Client, _ * expectedStatus.Ingress[0] = v1.LoadBalancerIngress{Hostname: hostname} status = makeLoadBalancerStatus(svc, nb) if !reflect.DeepEqual(status, expectedStatus) { - t.Errorf("expected status for %q annotated service to be %#v; got %#v", annLinodeHostnameOnlyIngress, expectedStatus, status) + t.Errorf("expected status for %q annotated service to be %#v; got %#v", annotations.AnnLinodeHostnameOnlyIngress, expectedStatus, status) } t.Setenv("LINODE_HOSTNAME_ONLY_INGRESS", "false") expectedStatus.Ingress[0] = v1.LoadBalancerIngress{Hostname: hostname} status = makeLoadBalancerStatus(svc, nb) if reflect.DeepEqual(status, expectedStatus) { - t.Errorf("expected status for %q annotated service to be %#v; got %#v", annLinodeHostnameOnlyIngress, expectedStatus, status) + t.Errorf("expected status for %q annotated service to be %#v; got %#v", annotations.AnnLinodeHostnameOnlyIngress, expectedStatus, status) } t.Setenv("LINODE_HOSTNAME_ONLY_INGRESS", "banana") expectedStatus.Ingress[0] = v1.LoadBalancerIngress{Hostname: hostname} status = makeLoadBalancerStatus(svc, nb) if reflect.DeepEqual(status, expectedStatus) { - t.Errorf("expected status for %q annotated service to be %#v; got %#v", annLinodeHostnameOnlyIngress, expectedStatus, status) + t.Errorf("expected status for %q annotated service to be %#v; got %#v", annotations.AnnLinodeHostnameOnlyIngress, expectedStatus, status) } os.Unsetenv("LINODE_HOSTNAME_ONLY_INGRESS") } @@ -2632,7 +2640,7 @@ func testCleanupDoesntCall(t *testing.T, client *linodego.Client, fakeAPI *fakeA svcAnn := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", - Annotations: map[string]string{annLinodeNodeBalancerID: strconv.Itoa(nb2.ID)}, + Annotations: map[string]string{annotations.AnnLinodeNodeBalancerID: strconv.Itoa(nb2.ID)}, }, } svc.Status.LoadBalancer = *makeLoadBalancerStatus(svc, nb1) @@ -2701,7 +2709,7 @@ func testUpdateLoadBalancerNoNodes(t *testing.T, client *linodego.Client, _ *fak svc.Status.LoadBalancer = *makeLoadBalancerStatus(svc, nodeBalancer) stubService(fakeClientset, svc) svc.ObjectMeta.SetAnnotations(map[string]string{ - annLinodeNodeBalancerID: strconv.Itoa(nodeBalancer.ID), + annotations.AnnLinodeNodeBalancerID: strconv.Itoa(nodeBalancer.ID), }) // setup done, test ensure/update @@ -2725,7 +2733,7 @@ func testGetNodeBalancerForServiceIDDoesNotExist(t *testing.T, client *linodego. Name: "test", UID: "foobar123", Annotations: map[string]string{ - annLinodeNodeBalancerID: bogusNodeBalancerID, + annotations.AnnLinodeNodeBalancerID: bogusNodeBalancerID, }, }, Spec: v1.ServiceSpec{ @@ -2769,7 +2777,7 @@ func testEnsureNewLoadBalancerWithNodeBalancerID(t *testing.T, client *linodego. Name: "testensure", UID: "foobar123", Annotations: map[string]string{ - annLinodeNodeBalancerID: strconv.Itoa(nodeBalancer.ID), + annotations.AnnLinodeNodeBalancerID: strconv.Itoa(nodeBalancer.ID), }, }, Spec: v1.ServiceSpec{ @@ -2813,8 +2821,8 @@ func testEnsureNewLoadBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI Name: "testensure", UID: "foobar123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", - annLinodePortConfigPrefix + "8443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, + annotations.AnnLinodeDefaultProtocol: "tcp", + annotations.AnnLinodePortConfigPrefix + "8443": `{ "protocol": "https", "tls-secret-name": "tls-secret"}`, }, }, Spec: v1.ServiceSpec{ @@ -2869,7 +2877,7 @@ func testGetLoadBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI) { Name: "test", UID: "foobar123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, Spec: v1.ServiceSpec{ @@ -2916,7 +2924,7 @@ func testGetLoadBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI) { Name: "notexists", UID: "notexists123", Annotations: map[string]string{ - annLinodeDefaultProtocol: "tcp", + annotations.AnnLinodeDefaultProtocol: "tcp", }, }, Spec: v1.ServiceSpec{ @@ -2978,7 +2986,7 @@ func Test_getPortConfigAnnotation(t *testing.T) { }{ { name: "Test single port annotation", - ann: map[string]string{annLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls", "protocol": "https" }`}, + ann: map[string]string{annotations.AnnLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls", "protocol": "https" }`}, expected: portConfigAnnotation{ TLSSecretName: "prod-app-tls", Protocol: "https", @@ -2988,8 +2996,8 @@ func Test_getPortConfigAnnotation(t *testing.T) { { name: "Test multiple port annotation", ann: map[string]string{ - annLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls", "protocol": "https" }`, - annLinodePortConfigPrefix + "80": `{ "protocol": "http" }`, + annotations.AnnLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls", "protocol": "https" }`, + annotations.AnnLinodePortConfigPrefix + "80": `{ "protocol": "http" }`, }, expected: portConfigAnnotation{ TLSSecretName: "prod-app-tls", @@ -3008,7 +3016,7 @@ func Test_getPortConfigAnnotation(t *testing.T) { { name: "Test invalid json", ann: map[string]string{ - annLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls" `, + annotations.AnnLinodePortConfigPrefix + "443": `{ "tls-secret-name": "prod-app-tls" `, }, expected: portConfigAnnotation{}, err: "unexpected end of JSON input", diff --git a/cloud/linode/node_controller.go b/cloud/linode/node_controller.go index 49880a97..e77eb20f 100644 --- a/cloud/linode/node_controller.go +++ b/cloud/linode/node_controller.go @@ -18,12 +18,15 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" ) type nodeController struct { sync.RWMutex - client Client + client client.Client instances *instances kubeclient kubernetes.Interface informer v1informers.NodeInformer @@ -34,7 +37,7 @@ type nodeController struct { queue workqueue.DelayingInterface } -func newNodeController(kubeclient kubernetes.Interface, client Client, informer v1informers.NodeInformer) *nodeController { +func newNodeController(kubeclient kubernetes.Interface, client client.Client, informer v1informers.NodeInformer) *nodeController { timeout := 300 if raw, ok := os.LookupEnv("LINODE_METADATA_TTL"); ok { if t, _ := strconv.Atoi(raw); t > 0 { @@ -124,7 +127,7 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { lastUpdate := s.LastMetadataUpdate(node.Name) - uuid, ok := node.Labels[annLinodeHostUUID] + uuid, ok := node.Labels[annotations.AnnLinodeHostUUID] if ok && time.Since(lastUpdate) < s.ttl { return nil } @@ -148,12 +151,12 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { } // It may be that the UUID has been set - if n.Labels[annLinodeHostUUID] == linode.HostUUID { + if n.Labels[annotations.AnnLinodeHostUUID] == linode.HostUUID { return nil } // Try to update the node - n.Labels[annLinodeHostUUID] = linode.HostUUID + n.Labels[annotations.AnnLinodeHostUUID] = linode.HostUUID _, err = s.kubeclient.CoreV1().Nodes().Update(ctx, n, metav1.UpdateOptions{}) return err }); err != nil {