From 52d6cb52c02a0c36de2dc2e41136907da0bc7786 Mon Sep 17 00:00:00 2001 From: Niladri Halder Date: Tue, 16 Jul 2024 12:09:31 +0530 Subject: [PATCH] tests: add tests for custom NodeAffinityLabels feature (#197) Signed-off-by: Niladri Halder --- tests/nodeAffinityLabels_cas_config_test.go | 147 ++++++++++++++++++++ tests/operations.go | 86 ++++++++++++ tests/pvc_cas_config_test.go | 99 +------------ 3 files changed, 237 insertions(+), 95 deletions(-) create mode 100644 tests/nodeAffinityLabels_cas_config_test.go diff --git a/tests/nodeAffinityLabels_cas_config_test.go b/tests/nodeAffinityLabels_cas_config_test.go new file mode 100644 index 00000000..d98f3619 --- /dev/null +++ b/tests/nodeAffinityLabels_cas_config_test.go @@ -0,0 +1,147 @@ +package tests + +import ( + ctx "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + pvc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolumeclaim" + sc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/storage/v1/storageclass" +) + +var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH 'NodeAffinityLabels' CAS-CONFIG ON STORAGECLASS", func() { + var ( + pvcNamePrefix = "pvc-nod-aff-lab" + scNamePrefix = "sc-nod-aff-lab" + deployNamePrefix = "busybox-nod-aff-lab" + deployment *appsv1.Deployment + scName string + pvcName string + pvcCapacity = "2Gi" + pvName string + scNodeAffinityLabelKeys = []string{"kubernetes.io/hostname", "kubernetes.io/os", "kubernetes.io/arch"} + ) + + When("an application with a PVC which has custom NodeAffinityLabels cas-config on the StorageClass, is created", func() { + It("should provision the volume", func() { + By("creating the StorageClass with custom NodeAffinityLabels", func() { + storageClass, err := sc.NewStorageClass( + sc.WithGenerateName(scNamePrefix), + sc.WithLabels(map[string]string{ + "openebs.io/test-sc": "true", + }), + sc.WithLocalPV(), + sc.WithHostpath(hostpathDir), + sc.WithNodeAffinityLabels(scNodeAffinityLabelKeys), + sc.WithVolumeBindingMode("WaitForFirstConsumer"), + sc.WithReclaimPolicy("Delete"), + ) + Expect(err).To( + BeNil(), + "while building StorageClass with name prefix {%s}", + scNamePrefix, + ) + storageClass, err = ops.SCClient.Create(ctx.TODO(), storageClass) + Expect(err).To( + BeNil(), + "while creating StorageClass with name prefix %s", + scNamePrefix, + ) + scName = storageClass.Name + }) + By("creating the PVC with the StorageClass "+scName, func() { + pvc, err := pvc.NewBuilder(). + WithGenerateName(pvcNamePrefix). + WithNamespace(namespaceObj.Name). + WithStorageClass(scName). + WithAccessModes([]corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}). + WithCapacity(pvcCapacity). + Build() + Expect(err).To( + BeNil(), + "while building PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvc, err = ops.PVCClient.WithNamespace(namespaceObj.Name).Create(ctx.TODO(), pvc) + Expect(err).To( + BeNil(), + "while creating PVC with name prefix %s in namespace %s", + pvcNamePrefix, + namespaceObj.Name, + ) + pvcName = pvc.Name + }) + By("creating a bound PV", func() { + deployment, err = ops.createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, + pvcName) + Expect(err).To( + BeNil(), + "while creating Deployment with name %s in namespace %s", + deployment.Name, + namespaceObj.Name, + ) + Expect(ops.IsPVCBoundEventually(namespaceObj.Name, pvcName)).To( + BeTrue(), + "while checking if the PVC %s in namespace %s was bound to a PV", + pvcName, namespaceObj.Name, + ) + Expect(ops.GetPodRunningCountEventually(namespaceObj.Name, "app="+deployNamePrefix, 1)).To(BeNumerically("==", + 1)) + Expect(Eventually(func() string { + pvName = ops.GetPVNameFromPVCName(namespaceObj.Name, pvcName) + return pvName + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + WithContext(ctx.TODO()). + Should(Not(BeEmpty()))).To(BeTrue()) + }) + + By("having the SC NodeAffinityLabels cas-config set correctly", func() { + nodeAffinityLabelKeys, err := ops.GetNodeAffinityLabelKeysFromPv(pvName) + Expect(err).To(BeNil(), "while getting NodeAffinityLabels from PV '%s'", pvName) + Expect(isLabelSelectorsEqual(scNodeAffinityLabelKeys, nodeAffinityLabelKeys)).To( + BeTrue(), + "while checking if PV %s had the NodeAffinityLabels requested on the SC %s", + scName, + ) + }) + }) + }) + + When("an application with a PV which has custom NodeAffinityLabels is deleted", func() { + It("should de-provision the volume", func() { + By("deleting the PV", func() { + podList, err := ops.PodClient.List(ctx.TODO(), metav1.ListOptions{LabelSelector: "app=" + deployNamePrefix}) + Expect(err).To(BeNil(), "while listing Pods for busybox application deployment") + Expect(len(podList.Items)).To(BeNumerically("==", 1)) + pod := &podList.Items[0] + err = ops.DeployClient.WithNamespace(namespaceObj.Name).Delete(ctx.TODO(), deployment.Name, &metav1.DeleteOptions{}) + Expect(err).To(BeNil(), "while deleting busybox application deployment") + Expect(ops.IsPodDeletedEventually(pod.Namespace, pod.Name)).To( + BeTrue(), + "while checking to see if the Pod %s in namespace %s for the busybox deployment is deleted", + pod.Name, pod.Namespace, + ) + + ops.DeletePersistentVolumeClaim(pvcName, namespaceObj.Name) + Expect(ops.IsPVCDeletedEventually(pvcName, namespaceObj.Name)).To( + BeTrue(), + "while checking if PVC %s in namespace %s is deleted", + pvcName, namespaceObj.Namespace, + ) + Expect(ops.IsPVDeletedEventually(pvName)).To( + BeTrue(), + "when checking to see if the underlying PV %s is deleted", + pvName, + ) + }) + }) + }) +}) diff --git a/tests/operations.go b/tests/operations.go index 5cf16530..14b04b73 100644 --- a/tests/operations.go +++ b/tests/operations.go @@ -1146,3 +1146,89 @@ func BuildPod(namespace, podName, pvcName string, labelselector map[string]strin ). Build() } + +// createDeploymentWhichConsumesHostpath creates a single-replica Deployment whose Pod consumes hostpath PVC. +func (ops *Operations) createDeploymentWhichConsumesHostpath(namePrefix, namespace, pvcName string) (*appsv1.Deployment, error) { + labelSelector := map[string]string{ + "app": namePrefix, + } + deployment, err := deploy.NewBuilder(). + WithGenerateName(namePrefix). + WithNamespace(namespace). + WithLabelsNew(labelSelector). + WithSelectorMatchLabelsNew(labelSelector). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew(labelSelector). + WithContainerBuildersNew( + container.NewBuilder(). + WithName("busybox"). + WithImage("busybox"). + WithCommandNew( + []string{ + "sleep", + "3600", + }, + ). + WithVolumeMountsNew( + []corev1.VolumeMount{ + { + Name: "demo-vol1", + MountPath: "/mnt/store1", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("demo-vol1"). + WithPVCSource(pvcName), + ), + ). + Build() + if err != nil { + return nil, err + } + + return ops.DeployClient.WithNamespace(namespace).Create(context.TODO(), deployment) +} + +// isLabelSelectorsEqual compares two arrays of label selector keys. +func isLabelSelectorsEqual(request, result []string) bool { + if len(request) != len(result) { + return false + } + + ch := make(chan struct{}, 2) + collectFrequency := func(labelKeys []string, freq *map[string]int) { + for _, elem := range labelKeys { + (*freq)[elem]++ + } + + ch <- struct{}{} + } + + // Maps to hold the frequency of strings in the string slices. + freqRequest := make(map[string]int) + freqResult := make(map[string]int) + + go collectFrequency(request, &freqRequest) + go collectFrequency(result, &freqResult) + + for i := 0; i < 2; i++ { + select { + case <-ch: + continue + } + } + + // Compare frequencies + for key, countRequest := range freqRequest { + countResult, ok := freqResult[key] + if !ok || countRequest != countResult { + return false + } + } + + return true +} diff --git a/tests/pvc_cas_config_test.go b/tests/pvc_cas_config_test.go index a9507a0b..379c41b9 100644 --- a/tests/pvc_cas_config_test.go +++ b/tests/pvc_cas_config_test.go @@ -7,106 +7,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" mayav1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" - "golang.org/x/net/context" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" - deploy "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/apps/v1/deployment" - "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container" pvc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolumeclaim" - pts "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/podtemplatespec" - k8svolume "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume" sc "github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/storage/v1/storageclass" ) -// createDeploymentWhichConsumesHostpath creates a single-replica Deployment whose Pod consumes hostpath PVC. -func createDeploymentWhichConsumesHostpath(namePrefix, namespace, pvcName string) (*appsv1.Deployment, error) { - labelSelector := map[string]string{ - "app": namePrefix, - } - deployment, err := deploy.NewBuilder(). - WithGenerateName(namePrefix). - WithNamespace(namespace). - WithLabelsNew(labelSelector). - WithSelectorMatchLabelsNew(labelSelector). - WithPodTemplateSpecBuilder( - pts.NewBuilder(). - WithLabelsNew(labelSelector). - WithContainerBuildersNew( - container.NewBuilder(). - WithName("busybox"). - WithImage("busybox"). - WithCommandNew( - []string{ - "sleep", - "3600", - }, - ). - WithVolumeMountsNew( - []corev1.VolumeMount{ - { - Name: "demo-vol1", - MountPath: "/mnt/store1", - }, - }, - ), - ). - WithVolumeBuilders( - k8svolume.NewBuilder(). - WithName("demo-vol1"). - WithPVCSource(pvcName), - ), - ). - Build() - if err != nil { - return nil, err - } - - return ops.DeployClient.WithNamespace(namespaceObj.Name).Create(context.TODO(), deployment) -} - -// isLabelSelectorsEqual compares two arrays of label selector keys. -func isLabelSelectorsEqual(request, result []string) bool { - if len(request) != len(result) { - return false - } - - ch := make(chan struct{}, 2) - collectFrequency := func(labelKeys []string, freq *map[string]int) { - for _, elem := range labelKeys { - (*freq)[elem]++ - } - - ch <- struct{}{} - } - - // Maps to hold the frequency of strings in the string slices. - freqRequest := make(map[string]int) - freqResult := make(map[string]int) - - go collectFrequency(request, &freqRequest) - go collectFrequency(result, &freqResult) - - for i := 0; i < 2; i++ { - select { - case <-ch: - continue - } - } - - // Compare frequencies - for key, countRequest := range freqRequest { - countResult, ok := freqResult[key] - if !ok || countRequest != countResult { - return false - } - } - - return true -} - var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH ADDITIVE CAS-CONFIGS ON PVC AND SC", func() { var ( pvcNamePrefix = "pvc-additive-cas-config" @@ -125,7 +34,7 @@ var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH ADDITIVE CAS-CONFIGS It("should provision the volume", func() { By("creating the StorageClass with cas-config", func() { storageClass, err := sc.NewStorageClass( - sc.WithGenerateName("sc-additive-cas-config"), + sc.WithGenerateName(scNamePrefix), sc.WithLabels(map[string]string{ "openebs.io/test-sc": "true", }), @@ -185,7 +94,7 @@ var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH ADDITIVE CAS-CONFIGS pvcName = pvc.Name }) By("creating a bound PV", func() { - deployment, err = createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, + deployment, err = ops.createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, pvcName) Expect(err).To( BeNil(), @@ -272,7 +181,7 @@ var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH CONFLICTING CAS-CONFI It("should provision the volume", func() { By("creating the StorageClass with cas-config", func() { storageClass, err := sc.NewStorageClass( - sc.WithGenerateName("sc-conflicting-cas-config"), + sc.WithGenerateName(scNamePrefix), sc.WithLabels(map[string]string{ "openebs.io/test-sc": "true", }), @@ -332,7 +241,7 @@ var _ = Describe("VOLUME PROVISIONING/DE-PROVISIONING WITH CONFLICTING CAS-CONFI pvcName = pvc.Name }) By("creating a bound PV", func() { - deployment, err = createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, + deployment, err = ops.createDeploymentWhichConsumesHostpath(deployNamePrefix, namespaceObj.Name, pvcName) Expect(err).To( BeNil(),