From 55fbab603882c283c8fe7fc3a73984eae96e1369 Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Thu, 28 Dec 2017 18:13:44 +0100 Subject: [PATCH 1/3] [DEVOPS-632] When updating the redis statefulset, add or delete exporter --- pkg/failover/client.go | 102 ++++++++------- pkg/failover/client_test.go | 241 ++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 43 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index b60173b4f..680c88de7 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -21,8 +21,9 @@ import ( // variables refering to the redis exporter port const ( - exporterPort = 9121 - exporterPortName = "http-metrics" + exporterPort = 9121 + exporterPortName = "http-metrics" + exporterContainerName = "redis-exporter" ) const ( @@ -645,47 +646,7 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro } if rf.Spec.Redis.Exporter { - exporter := v1.Container{ - Name: "redis-exporter", - Image: exporterImage, - ImagePullPolicy: "Always", - Ports: []v1.ContainerPort{ - v1.ContainerPort{ - Name: "metrics", - ContainerPort: exporterPort, - Protocol: v1.ProtocolTCP, - }, - }, - ReadinessProbe: &v1.Probe{ - InitialDelaySeconds: 10, - TimeoutSeconds: 3, - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/", - Port: intstr.FromString("metrics"), - }, - }, - }, - LivenessProbe: &v1.Probe{ - TimeoutSeconds: 3, - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/", - Port: intstr.FromString("metrics"), - }, - }, - }, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("300m"), - v1.ResourceMemory: resource.MustParse("300Mi"), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("150Mi"), - }, - }, - } + exporter := createRedisExporterContainer() redisStatefulset.Spec.Template.Spec.Containers = append(redisStatefulset.Spec.Template.Spec.Containers, exporter) } @@ -818,6 +779,17 @@ func (r *RedisFailoverKubeClient) UpdateRedisStatefulset(rf *RedisFailover) erro oldSS.Spec.Template.Spec.Containers[0].Resources = getRedisResources(rf.Spec) oldSS.Spec.Template.Spec.Containers[0].Image = getRedisImage(rf) + if rf.Spec.Redis.Exporter { + exporter := createRedisExporterContainer() + oldSS.Spec.Template.Spec.Containers = append(oldSS.Spec.Template.Spec.Containers, exporter) + } else { + for pos, container := range oldSS.Spec.Template.Spec.Containers { + if container.Name == exporterContainerName { + oldSS.Spec.Template.Spec.Containers = append(oldSS.Spec.Template.Spec.Containers[:pos], oldSS.Spec.Template.Spec.Containers[pos+1:]...) + } + } + } + if _, err := r.Client.AppsV1beta1().StatefulSets(namespace).Update(oldSS); err != nil { return err } @@ -955,3 +927,47 @@ func generateResourceList(cpu string, memory string) v1.ResourceList { } return resources } + +func createRedisExporterContainer() v1.Container { + return v1.Container{ + Name: exporterContainerName, + Image: exporterImage, + ImagePullPolicy: "Always", + Ports: []v1.ContainerPort{ + v1.ContainerPort{ + Name: "metrics", + ContainerPort: exporterPort, + Protocol: v1.ProtocolTCP, + }, + }, + ReadinessProbe: &v1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 3, + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/", + Port: intstr.FromString("metrics"), + }, + }, + }, + LivenessProbe: &v1.Probe{ + TimeoutSeconds: 3, + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/", + Port: intstr.FromString("metrics"), + }, + }, + }, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("150Mi"), + }, + }, + } +} diff --git a/pkg/failover/client_test.go b/pkg/failover/client_test.go index f6812fcb2..f1c7556e3 100644 --- a/pkg/failover/client_test.go +++ b/pkg/failover/client_test.go @@ -1659,6 +1659,247 @@ func TestUpdateRedisStatefulsetError(t *testing.T) { assert.Error(err) } +func TestUpdateRedisStatefulsetWithUpdate(t *testing.T) { + assert := assert.New(t) + + replicas := int32(3) + replicasUpdated := int32(4) + called := false + cpu := "200m" + memory := "200Mi" + cpuQuantityOriginal, _ := resource.ParseQuantity("100m") + memoryQuantityOriginal, _ := resource.ParseQuantity("100Mi") + cpuQuantityRequired, _ := resource.ParseQuantity(cpu) + memoryQuantityRequired, _ := resource.ParseQuantity(memory) + var updatedRequests v1.ResourceRequirements + + requiredRequests := v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityRequired, + v1.ResourceMemory: memoryQuantityRequired, + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityRequired, + v1.ResourceMemory: memoryQuantityRequired, + }, + } + + exporterExists := false + + // Create a faked K8S client + client := &fake.Clientset{} + client.Fake.AddReactor("get", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) { + r := replicas + if called { + r = replicasUpdated + } + statefulset := &v1beta1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: redisName, + Namespace: namespace, + }, + Status: v1beta1.StatefulSetStatus{ + ReadyReplicas: r, + UpdatedReplicas: r, + }, + Spec: v1beta1.StatefulSetSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: redisName, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityOriginal, + v1.ResourceMemory: memoryQuantityOriginal, + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityOriginal, + v1.ResourceMemory: memoryQuantityOriginal, + }, + }, + }, + }, + }, + }, + }, + } + called = true + return true, statefulset, nil + }) + client.Fake.AddReactor("update", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) { + updateAction := action.(k8stesting.UpdateAction) + statefulset := updateAction.GetObject().(*v1beta1.StatefulSet) + for _, container := range statefulset.Spec.Template.Spec.Containers { + if container.Name == redisName { + updatedRequests = container.Resources + } + if container.Name == "redis-exporter" { + exporterExists = true + } + } + return true, nil, nil + }) + + mc := &mocks.Clock{} + mc.On("NewTicker", mock.Anything). + Once().Return(time.NewTicker(1)) + r := failover.NewRedisFailoverKubeClient(client, mc, log.Nil) + + redisFailover := &failover.RedisFailover{ + Metadata: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: failover.RedisFailoverSpec{ + Redis: failover.RedisSettings{ + Replicas: replicasUpdated, + Resources: failover.RedisFailoverResources{ + Limits: failover.CPUAndMem{ + CPU: cpu, + Memory: memory, + }, + Requests: failover.CPUAndMem{ + CPU: cpu, + Memory: memory, + }, + }, + Exporter: true, + }, + Sentinel: failover.SentinelSettings{ + Replicas: int32(3), + }, + }, + } + + err := r.UpdateRedisStatefulset(redisFailover) + assert.NoError(err) + assert.Equal(requiredRequests, updatedRequests, "Requests are not equal as updated") + assert.True(exporterExists, "Redis-exporter should exist") +} + +func TestUpdateRedisStatefulsetWithoutUpdate(t *testing.T) { + assert := assert.New(t) + + replicas := int32(3) + replicasUpdated := int32(4) + called := false + cpu := "200m" + memory := "200Mi" + cpuQuantityOriginal, _ := resource.ParseQuantity("100m") + memoryQuantityOriginal, _ := resource.ParseQuantity("100Mi") + cpuQuantityRequired, _ := resource.ParseQuantity(cpu) + memoryQuantityRequired, _ := resource.ParseQuantity(memory) + var updatedRequests v1.ResourceRequirements + + requiredRequests := v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityRequired, + v1.ResourceMemory: memoryQuantityRequired, + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityRequired, + v1.ResourceMemory: memoryQuantityRequired, + }, + } + + exporterExists := false + + // Create a faked K8S client + client := &fake.Clientset{} + client.Fake.AddReactor("get", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) { + r := replicas + if called { + r = replicasUpdated + } + statefulset := &v1beta1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: redisName, + Namespace: namespace, + }, + Status: v1beta1.StatefulSetStatus{ + ReadyReplicas: r, + UpdatedReplicas: r, + }, + Spec: v1beta1.StatefulSetSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: redisName, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityOriginal, + v1.ResourceMemory: memoryQuantityOriginal, + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: cpuQuantityOriginal, + v1.ResourceMemory: memoryQuantityOriginal, + }, + }, + }, + v1.Container{ + Name: "redis-exporter", + }, + }, + }, + }, + }, + } + called = true + return true, statefulset, nil + }) + client.Fake.AddReactor("update", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) { + updateAction := action.(k8stesting.UpdateAction) + statefulset := updateAction.GetObject().(*v1beta1.StatefulSet) + for _, container := range statefulset.Spec.Template.Spec.Containers { + if container.Name == redisName { + updatedRequests = container.Resources + } + if container.Name == "redis-exporter" { + exporterExists = true + } + } + return true, nil, nil + }) + + mc := &mocks.Clock{} + mc.On("NewTicker", mock.Anything). + Once().Return(time.NewTicker(1)) + r := failover.NewRedisFailoverKubeClient(client, mc, log.Nil) + + redisFailover := &failover.RedisFailover{ + Metadata: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: failover.RedisFailoverSpec{ + Redis: failover.RedisSettings{ + Replicas: replicasUpdated, + Resources: failover.RedisFailoverResources{ + Limits: failover.CPUAndMem{ + CPU: cpu, + Memory: memory, + }, + Requests: failover.CPUAndMem{ + CPU: cpu, + Memory: memory, + }, + }, + Exporter: false, + }, + Sentinel: failover.SentinelSettings{ + Replicas: int32(3), + }, + }, + } + + err := r.UpdateRedisStatefulset(redisFailover) + assert.NoError(err) + assert.Equal(requiredRequests, updatedRequests, "Requests are not equal as updated") + assert.False(exporterExists, "Redis-exporter should not exist") +} + func TestUpdateRedisStatefulset(t *testing.T) { assert := assert.New(t) From 33d422b9b7ddab5103a55b419f92dc7112eb354e Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Fri, 29 Dec 2017 12:08:59 +0100 Subject: [PATCH 2/3] [DEVOPS-632] Bump version to 0.1.3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3c2cea827..191a0699e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # The following are targers that do not exist in the filesystem as real files and should be always executed by make .PHONY: default build deps-development docker-build shell run image unit-test test generate go-generate get-deps update-deps -VERSION := 0.1.2 +VERSION := 0.1.3 # Name of this service/application SERVICE_NAME := redis-operator From b0e546934b4140a3118c7eec466f320a30feae75 Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Fri, 29 Dec 2017 12:55:44 +0100 Subject: [PATCH 3/3] [DEVOPS-632] Change default exporter resources/limits --- pkg/failover/client.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index 680c88de7..fafd440c5 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -21,9 +21,13 @@ import ( // variables refering to the redis exporter port const ( - exporterPort = 9121 - exporterPortName = "http-metrics" - exporterContainerName = "redis-exporter" + exporterPort = 9121 + exporterPortName = "http-metrics" + exporterContainerName = "redis-exporter" + exporterDefaultRequestCPU = "25m" + exporterDefaultLimitCPU = "50m" + exporterDefaultRequestMemory = "50Mi" + exporterDefaultLimitMemory = "100Mi" ) const ( @@ -961,12 +965,12 @@ func createRedisExporterContainer() v1.Container { }, Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("300m"), - v1.ResourceMemory: resource.MustParse("300Mi"), + v1.ResourceCPU: resource.MustParse(exporterDefaultLimitCPU), + v1.ResourceMemory: resource.MustParse(exporterDefaultLimitMemory), }, Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("150Mi"), + v1.ResourceCPU: resource.MustParse(exporterDefaultRequestCPU), + v1.ResourceMemory: resource.MustParse(exporterDefaultRequestMemory), }, }, }