From cbb6dc2a95cb5b67d6e08049b705f80e742c97bf Mon Sep 17 00:00:00 2001 From: sai chaithanya Date: Fri, 9 Jul 2021 21:22:41 +0530 Subject: [PATCH] feat(node affinity): add support to specify node affinity rules of NFS Server (#59) * feat(node affinity): add support to specify node affinity rules of NFS Server This commit adds support to specify node affinity rules via NFS-Provisioner ENV to schedule scheduling NFS Server on set of nodes. **How to use?**: - Add 'OPENEBS_IO_NFS_SERVER_NODE_AFFINITY' ENV in NFS-Provisioner deployment in following manner: ```sh - name: OPENEBS_IO_NFS_SERVER_NODE_AFFINITY value: "kubernetes.io/hostname:[172.17.0.1],kubernetes.io/os:[linux]" ``` - To schedule NFS Server instance on storage & nfs nodes ```sh - name: OPENEBS_IO_NFS_SERVER_NODE_AFFINITY value: "kubernetes.io/storage,kubernetes.io/nfs-node" ``` Ex: Propagation to NFS Server deployment ```yaml ... ... ... nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/storage-node operator: Exists - key: kubernetes.io/arch operator: Exists ``` **How it is propagated to NFS Server instance**: - During boot-up time of provisioner instance, provisioner will read OPENEBS_IO_NFS_SERVER_NODE_AFFINITY ENV then parse affinity rules and store them under the affinity rules in form of Go structure[in-memory]. - When volume is provisioned NFS-Provisioner will propagate this affinity rules to NFS Server instance. Example propagation: ```yaml ... ... ... affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - 172.17.0.1 - key: kubernetes.io/os operator: In values: - linux ``` * add integration test case to verify node affinity rules * helm chart changes and fixes test cases Signed-off-by: mittachaitu --- deploy/helm/charts/Chart.yaml | 2 +- deploy/helm/charts/README.md | 1 + deploy/helm/charts/templates/deployment.yaml | 13 +- deploy/helm/charts/values.yaml | 14 +- deploy/kubectl/openebs-nfs-provisioner.yaml | 34 ++- go.mod | 1 + go.sum | 1 + .../v1/podtemplatespec/podtemplatespec.go | 52 ++++ provisioner/env.go | 7 + provisioner/helper_kernel_nfs_server.go | 1 + provisioner/node_affinity.go | 194 +++++++++++++++ provisioner/node_affinity_test.go | 215 +++++++++++++++++ provisioner/provisioner.go | 47 +++- provisioner/types.go | 70 +++++- tests/k8s_utils.go | 26 +- tests/node_affinity_test.go | 224 ++++++++++++++++++ 16 files changed, 874 insertions(+), 28 deletions(-) create mode 100644 provisioner/node_affinity.go create mode 100644 provisioner/node_affinity_test.go create mode 100644 tests/node_affinity_test.go diff --git a/deploy/helm/charts/Chart.yaml b/deploy/helm/charts/Chart.yaml index 3791f59..24ba7b3 100644 --- a/deploy/helm/charts/Chart.yaml +++ b/deploy/helm/charts/Chart.yaml @@ -4,7 +4,7 @@ description: Helm chart for OpenEBS Dynamic NFS PV. For instructions to install type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.4.1 +version: 0.4.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.4.0 diff --git a/deploy/helm/charts/README.md b/deploy/helm/charts/README.md index a63a57d..f37b114 100644 --- a/deploy/helm/charts/README.md +++ b/deploy/helm/charts/README.md @@ -128,6 +128,7 @@ helm install openebs-nfs openebs-nfs/nfs-provisioner --namespace openebs --creat | `nfsProvisioner.securityContext` | Security context for container | `""` | | `nfsProvisioner.tolerations` | NFS Provisioner pod toleration values | `""` | | `nfsProvisioner.nfsServerNamespace` | NFS server namespace | `"openebs"` | +| `nfsProvisioner.nfsServerNodeAffinity` | NFS Server node affinity rules | `""` | | `nfsStorageClass.backendStorageClass` | StorageClass to be used to provision the backend volume. If not specified, the default StorageClass is used. | `""` | | `nfsStorageClass.isDefaultClass` | Make 'openebs-kernel-nfs' the default StorageClass | `"false"` | | `nfsStorageClass.reclaimPolicy` | ReclaimPolicy for NFS PVs | `"Delete"` | diff --git a/deploy/helm/charts/templates/deployment.yaml b/deploy/helm/charts/templates/deployment.yaml index dcaf01f..eaa67f7 100644 --- a/deploy/helm/charts/templates/deployment.yaml +++ b/deploy/helm/charts/templates/deployment.yaml @@ -87,7 +87,18 @@ spec: - name: OPENEBS_IO_NFS_SERVER_NS value: {{ .Values.nfsProvisioner.nfsServerNamespace }} {{- end }} - + # OPENEBS_IO_NFS_SERVER_NODE_AFFINITY defines the node affinity rules to place NFS Server + # instance. It accepts affinity rules in multiple ways: + # - If NFS Server needs to be placed on storage nodes as well as only in + # zone-1 & zone-2 then value can be: + # value: "kubernetes.io/zone:[zone-1,zone-2],kubernetes.io/storage-node". + # - If NFS Server needs to be placed only on storage nodes & nfs nodes then + # value can be: + # value: "kubernetes.io/storage-node,kubernetes.io/nfs-node" + {{- if .Values.nfsProvisioner.nfsServerNodeAffinity }} + - name: OPENEBS_IO_NFS_SERVER_NODE_AFFINITY + value: "{{ .Values.nfsProvisioner.nfsServerNodeAffinity }}" + {{- end }} # Process name used for matching is limited to the 15 characters # present in the pgrep output. # So fullname can't be used here with pgrep (>15 chars).A regular expression diff --git a/deploy/helm/charts/values.yaml b/deploy/helm/charts/values.yaml index 265ba12..a01cbd1 100644 --- a/deploy/helm/charts/values.yaml +++ b/deploy/helm/charts/values.yaml @@ -66,9 +66,17 @@ nfsProvisioner: healthCheck: initialDelaySeconds: 30 periodSeconds: 60 - # namespace in which nfs server objects should be created - # By default, nfs provisioner will create these resources in nfs provisioner's namespace - # nfsServerNamespace: openebs + # namespace in which nfs server objects should be created + # By default, nfs provisioner will create these resources in nfs provisioner's namespace + # nfsServerNamespace: openebs + # + # nfsServerNodeAffinity defines the node affinity rules to place NFS Server + # instance. It accepts affinity rules in multiple ways: + # - If NFS Server needs to be placed on storage nodes as well as only in + # zone-1 & zone-2 then value can be: "kubernetes.io/zone:[zone-1,zone-2],kubernetes.io/storage-node". + # - If NFS Server needs to be placed only on storage nodes & nfs nodes then + # value can be: "kubernetes.io/storage-node,kubernetes.io/nfs-node" + # nfsServerNodeAffinity: "kubernetes.io/storage-node,kubernetes.io/nfs-node" nfsStorageClass: name: openebs-kernel-nfs diff --git a/deploy/kubectl/openebs-nfs-provisioner.yaml b/deploy/kubectl/openebs-nfs-provisioner.yaml index 5a5d25b..212039a 100644 --- a/deploy/kubectl/openebs-nfs-provisioner.yaml +++ b/deploy/kubectl/openebs-nfs-provisioner.yaml @@ -92,16 +92,26 @@ spec: imagePullPolicy: IfNotPresent image: openebs/provisioner-nfs:ci env: - # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s - # based on this address. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_K8S_MASTER - # value: "http://10.128.0.12:8080" - # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s - # based on this config. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_KUBE_CONFIG - # value: "/home/ubuntu/.kube/config" + # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s + # based on this address. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + # - name: OPENEBS_IO_K8S_MASTER + # value: "http://10.128.0.12:8080" + # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s + # based on this config. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + # - name: OPENEBS_IO_KUBE_CONFIG + # value: "/home/ubuntu/.kube/config" + # OPENEBS_IO_NFS_SERVER_NODE_AFFINITY defines the node affinity rules to place NFS Server + # instance. It accepts affinity rules in multiple ways: + # - If NFS Server needs to be placed on storage nodes as well as only in + # zone-1 & zone-2 then value can be: + # value: "kubernetes.io/zone:[zone-1,zone-2],kubernetes.io/storage-node". + # - If NFS Server needs to be placed only on storage nodes & nfs nodes then + # value can be: + # value: "kubernetes.io/storage-node,kubernetes.io/nfs-node" + # - name: OPENEBS_IO_NFS_SERVER_NODE_AFFINITY + # value: "kubernetes.io/storage-node,kubernetes.io/nfs-node" - name: NODE_NAME valueFrom: fieldRef: @@ -158,8 +168,8 @@ metadata: cas.openebs.io/config: | - name: NFSServerType value: "kernel" - #- name: BackendStorageClass - # value: "openebs-hostpath" + - name: BackendStorageClass + value: "openebs-hostpath" # LeaseTime defines the renewl period(in seconds) for client state #- name: LeaseTime # value: 30 diff --git a/go.mod b/go.mod index ad21aaf..8fd317f 100644 --- a/go.mod +++ b/go.mod @@ -42,5 +42,6 @@ require ( k8s.io/apimachinery v0.17.3 k8s.io/client-go v11.0.0+incompatible k8s.io/klog v1.0.0 + k8s.io/kubernetes v1.17.3 sigs.k8s.io/sig-storage-lib-external-provisioner v4.1.0+incompatible ) diff --git a/go.sum b/go.sum index dce2b77..3e0e97c 100644 --- a/go.sum +++ b/go.sum @@ -860,6 +860,7 @@ k8s.io/kube-proxy v0.17.3/go.mod h1:ds8R8bUYPWtQlspC47Sff7o5aQhWDsv6jpQJATDuqaQ= k8s.io/kube-scheduler v0.17.3/go.mod h1:36HgrrPqzK+rOLTRtDG//b89KjrAZqFI4PXOpdH351M= k8s.io/kubectl v0.17.3/go.mod h1:NUn4IBY7f7yCMwSop2HCXlw/MVYP4HJBiUmOR3n9w28= k8s.io/kubelet v0.17.3/go.mod h1:Nh8owUHZcUXtnDAtmGnip36Nw+X6c4rbmDQlVyIhwMQ= +k8s.io/kubernetes v1.17.3 h1:zWCppkLfHM+hoLqfbsrQ0cJnYw+4vAvedI92oQnjo/Q= k8s.io/kubernetes v1.17.3/go.mod h1:gt28rfzaskIzJ8d82TSJmGrJ0XZD0BBy8TcQvTuCI3w= k8s.io/legacy-cloud-providers v0.17.3/go.mod h1:ujZML5v8efVQxiXXTG+nck7SjP8KhMRjUYNIsoSkYI0= k8s.io/metrics v0.17.3/go.mod h1:HEJGy1fhHOjHggW9rMDBJBD3YuGroH3Y1pnIRw9FFaI= diff --git a/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go b/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go index d735f98..6cfef7c 100644 --- a/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go +++ b/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go @@ -273,6 +273,58 @@ func (b *Builder) WithAffinity(affinity *corev1.Affinity) *Builder { return b } +// WithNodeAffinityMatchExpressions sets matchexpressions under +// nodeAffinity +// NOTE: If nil is passed then match expressions will not be +// propogated to node affinity. +// CAUTION: Don't invoke WithAffinity func after calling this function +// It will overwrite MatchExpression +func (b *Builder) WithNodeAffinityMatchExpressions( + mExpressions []corev1.NodeSelectorRequirement) *Builder { + if len(mExpressions) == 0 { + return b + } + + if b.podtemplatespec.Object.Spec.Affinity == nil { + b.podtemplatespec.Object.Spec.Affinity = &corev1.Affinity{} + } + if b.podtemplatespec.Object.Spec.Affinity.NodeAffinity == nil { + b.podtemplatespec.Object.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + if b.podtemplatespec. + Object. + Spec. + Affinity. + NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution == nil { + b.podtemplatespec. + Object. + Spec. + Affinity. + NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} + } + + b.podtemplatespec. + Object. + Spec. + Affinity. + NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution. + NodeSelectorTerms = append(b.podtemplatespec. + Object. + Spec. + Affinity. + NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution. + NodeSelectorTerms, + corev1.NodeSelectorTerm{ + MatchExpressions: mExpressions, + }, + ) + return b +} + // WithTolerationsByValue sets pod toleration. // If provided tolerations argument is empty it does not complain. func (b *Builder) WithTolerationsByValue(tolerations ...corev1.Toleration) *Builder { diff --git a/provisioner/env.go b/provisioner/env.go index 49aa6d6..a91f3f4 100644 --- a/provisioner/env.go +++ b/provisioner/env.go @@ -51,6 +51,9 @@ const ( // NFSServerNamespace defines the namespace for nfs server objects // Default value is menv.OpenEBSNamespace(operator namespace) NFSServerNamespace menv.ENVKey = "OPENEBS_IO_NFS_SERVER_NS" + + // NodeAffinityKey holds the env name representing Node affinity rules + NodeAffinityKey menv.ENVKey = "OPENEBS_IO_NFS_SERVER_NODE_AFFINITY" ) var ( @@ -87,3 +90,7 @@ func getOpenEBSServiceAccountName() string { func getNFSServerImage() string { return menv.GetOrDefault(NFSServerImageKey, string(NFSServerDefaultImage)) } + +func getNfsServerNodeAffinity() string { + return menv.Get(NodeAffinityKey) +} diff --git a/provisioner/helper_kernel_nfs_server.go b/provisioner/helper_kernel_nfs_server.go index 8ea2e83..504b945 100644 --- a/provisioner/helper_kernel_nfs_server.go +++ b/provisioner/helper_kernel_nfs_server.go @@ -217,6 +217,7 @@ func (p *Provisioner) createDeployment(nfsServerOpts *KernelNFSServerOptions) er WithSecurityContext(&corev1.PodSecurityContext{ FSGroup: nfsServerOpts.fsGroup, }). + WithNodeAffinityMatchExpressions(p.nodeAffinity.MatchExpressions). WithContainerBuildersNew( container.NewBuilder(). WithName("nfs-server"). diff --git a/provisioner/node_affinity.go b/provisioner/node_affinity.go new file mode 100644 index 0000000..46b20d5 --- /dev/null +++ b/provisioner/node_affinity.go @@ -0,0 +1,194 @@ +/* +Copyright 2021 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + "regexp" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +// getNodeAffinityRules fetchs node affinity rules from +// environment value +func getNodeAffinityRules() NodeAffinity { + var nodeAffinity NodeAffinity + + affinityValue := getNfsServerNodeAffinity() + if affinityValue == "" { + return nodeAffinity + } + + rules := strings.Split(affinityValue, "],") + for _, rule := range rules { + nodeAffinity.MatchExpressions = append(nodeAffinity.MatchExpressions, getOneOrMoreNodeSelectorRequirements(rule)...) + } + + return nodeAffinity +} + +// getOneOrMoreNodeSelectorRequirements can take one or more node affinity requirements +// as string and convert them to structured form of Requirements +// Ex: +// Case1 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/zone:[zone-1,zone-2,zone-3] +// +// Return value: +// - key: kubernetes.io/storage-node +// operator: Exists +// - key: kubernetes.io/nfs-node +// operator: Exists +// - key: kubernetes.io/zone +// operator: In +// values: +// - zone-1 +// - zone-2 +// - zone-3 +// +// Case2 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/linux-amd64 +// +// Return value: +// - key: kubernetes.io/storage-node +// operator: Exists +// - key: kubernetes.io/nfs-node +// operator: Exists +// - key: kubernetes.io/linux-amd64 +// operator: Exists +// +// Case3 - Input argument: kubernetes.io/zone:[zone-1,zone-2] +// +// Return value: +// - key: kubernetes.io/zone +// operator: In +// values: +// - zone-1 +// - zone-2 +func getOneOrMoreNodeSelectorRequirements( + requirementsAsValue string) []corev1.NodeSelectorRequirement { + var nodeRequirements []corev1.NodeSelectorRequirement + var complexReq corev1.NodeSelectorRequirement + // isComplexRequirement will be true when input is: ,,:[value1, value2] + // NOTE: Valued key-value pair will be always at end + isComplexRequirement := regexp.MustCompile(`.*,+.*:\[.*`).FindString(requirementsAsValue) != "" + + if isComplexRequirement { + matchingString := getRightMostMatchingString(regexp.MustCompile(`,.*:\[.*`), requirementsAsValue) + // If input argument is Case 1 + if matchingString != "" { + matchingIndex := strings.LastIndex(requirementsAsValue, matchingString) + complexReq = getNodeSelectorRequirement(requirementsAsValue[matchingIndex:]) + requirementsAsValue = requirementsAsValue[:matchingIndex] + } + } + + // After processing complex now we will left with two cases + // C1: , + // C2: :[value2] --- Original Case3 + if strings.ContainsRune(requirementsAsValue, rune('[')) { + // Case3 + nodeRequirements = append(nodeRequirements, getNodeSelectorRequirement(requirementsAsValue)) + } else { + // Case2 + for _, req := range strings.Split(requirementsAsValue, ",") { + if strings.TrimSpace(req) != "" { + nodeRequirements = append(nodeRequirements, getNodeSelectorRequirement(req)) + } + } + if isComplexRequirement { + nodeRequirements = append(nodeRequirements, complexReq) + } + } + + return nodeRequirements +} + +// getNodeSelectorRequirement converts requirement from plain +// string to corev1.NodeSelectorRequirement +// +// Example: kubernetes.io/hostName:[z1-host1,z2-host1,z3-host1] value convert as below +// +// key: kubernetes.io/hostName +// operator: "In" +// values: +// - z1-host1 +// - z2-host1 +// - z3-host1 +// +// Example: kubernetes.io/hostName:[region-1,region-2 value convert as below +// +// key: kubernetes.io/hostName +// operator: "In" +// values: +// - region-1 +// - region-2 +// +// Example: kubernetes.io/storage-node +// +// key: kubernetes.io/storage-node +// operator: "Exists" +func getNodeSelectorRequirement(reqAsValue string) corev1.NodeSelectorRequirement { + var nsRequirement corev1.NodeSelectorRequirement + keyValues := strings.Split(reqAsValue, ":") + // Key will always exist in given ENV + nsRequirement.Key = strings.TrimSpace(keyValues[0]) + nsRequirement.Operator = corev1.NodeSelectorOpExists + + // If there exist more than one value + if len(keyValues) > 1 { + valueList := strings.Split( + strings.TrimSpace( + strings.TrimLeft( + strings.TrimRight(keyValues[1], "]"), + "["), + ), + ",") + + // If user mentioned list of values + if len(valueList) > 1 || (len(valueList) == 1 && strings.TrimSpace(valueList[0]) != "") { + nsRequirement.Operator = corev1.NodeSelectorOpIn + nsRequirement.Values = valueList + } + } + + return nsRequirement +} + +// getRightMostMatchingString will return right must matching string +// which satisfies given pattern +// Example: +// - Fetch last pattern matching on string +// Pattern: {,.*:\[.*} string: "key1,key2,key3:[v1, v2, v3]" +// Return value: key3:[v1, v2, v3] +func getRightMostMatchingString(regex *regexp.Regexp, value string) string { + loc := regex.FindStringIndex(value) + if len(loc) == 0 { + // given value is not satisfying regular expression + return "" + } + value = value[loc[0]:] + if value[0] == ',' && len(value) > 1 { + value = value[1:] + } + rightMostMatchingString := getRightMostMatchingString(regex, value) + + // If substring matching to regular expression is found then return + // right most index + if rightMostMatchingString != "" { + return rightMostMatchingString + } + // else return starting location + return value +} diff --git a/provisioner/node_affinity_test.go b/provisioner/node_affinity_test.go new file mode 100644 index 0000000..aa6a007 --- /dev/null +++ b/provisioner/node_affinity_test.go @@ -0,0 +1,215 @@ +/* +Copyright 2021 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + "os" + "reflect" + "regexp" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestGetNodeAffinityRules(t *testing.T) { + tests := map[string]struct { + envValue string + expectedAffinity NodeAffinity + }{ + "when there is only single topology without values": { + envValue: "kubernetes.io/storage-node:[]", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies with values": { + envValue: "kubernetes.io/storage-node:[],kubernetes.io/zone:[zone-a,zone-b,zone-c]," + + "kubernetes.io/region:[region-1],kubernetes.io/nfs-node:[]", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/zone", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"zone-a", "zone-b", "zone-c"}, + }, + { + Key: "kubernetes.io/region", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"region-1"}, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies without values": { + envValue: "kubernetes.io/storage-node:[],kubernetes.io/nfs-node:[]", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies without values & []": { + envValue: "kubernetes.io/storage-node ,kubernetes.io/nfs-node", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies without empty values([])": { + envValue: "kubernetes.io/nfs-node,kubernetes.io/storage-node,kubernetes.io/zone:[zone-a,zone-b,zone-c]," + + "kubernetes.io/region:[region-1],kubernetes.io/nfs-node", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/zone", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"zone-a", "zone-b", "zone-c"}, + }, + { + Key: "kubernetes.io/region", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"region-1"}, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies without values but one of them has empty": { + envValue: "kubernetes.io/storage-node,kubernetes.io/nfs-node:[]", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + "when there are multiple topologies with empty and without values": { + envValue: "kubernetes.io/storage-node:[],kubernetes.io/nfs-node", + expectedAffinity: NodeAffinity{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/storage-node", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "kubernetes.io/nfs-node", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, + } + + for name, test := range tests { + os.Setenv(string(NodeAffinityKey), test.envValue) + gotNodeAffinityRules := getNodeAffinityRules() + if !reflect.DeepEqual(gotNodeAffinityRules.MatchExpressions, test.expectedAffinity.MatchExpressions) { + t.Errorf( + "%q test got failed expected %v but got %v", + name, + test.expectedAffinity.MatchExpressions, + gotNodeAffinityRules.MatchExpressions, + ) + } + + os.Unsetenv(string(NodeAffinityKey)) + } +} + +func TestGetRightMostMatchingIndex(t *testing.T) { + tests := map[string]struct { + regexp *regexp.Regexp + str string + expectedString string + }{ + "When repitative pattern exist twice": { + regexp: regexp.MustCompile(`,+.*:\[.*`), + str: "key1,key2,key3:[v1,v2,v3]", + expectedString: "key3:[v1,v2,v3]", + }, + "When pattern exist exactly once": { + regexp: regexp.MustCompile(`,+.*:\[.*`), + str: ",key3:[v1,v2,v3]", + expectedString: "key3:[v1,v2,v3]", + }, + "When pattern matches more than twice": { + regexp: regexp.MustCompile(`,+.*:\[.*`), + str: "key1,key2,key3,key4:[v1,v2]", + expectedString: "key4:[v1,v2]", + }, + "When pattern does not match with given string": { + regexp: regexp.MustCompile(`abcd`), + str: "openebs", + expectedString: "", + }, + } + for name, test := range tests { + name := name + test := test + t.Run(name, func(t *testing.T) { + gotString := getRightMostMatchingString(test.regexp, test.str) + if gotString != test.expectedString { + t.Errorf("%q test failed expected: %q but got %q", name, test.expectedString, gotString) + } + }) + } +} diff --git a/provisioner/provisioner.go b/provisioner/provisioner.go index bb43c07..ff6598a 100644 --- a/provisioner/provisioner.go +++ b/provisioner/provisioner.go @@ -47,6 +47,9 @@ import ( analytics "github.com/openebs/maya/pkg/usage" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + kubeinformers "k8s.io/client-go/informers" + listersv1 "k8s.io/client-go/listers/core/v1" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -60,6 +63,8 @@ func NewProvisioner(stopCh chan struct{}, kubeClient *clientset.Clientset) (*Pro if len(strings.TrimSpace(namespace)) == 0 { return nil, fmt.Errorf("Cannot start Provisioner: failed to get namespace") } + kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, 0) + k8sNodeInformer := kubeInformerFactory.Core().V1().Nodes().Informer() nfsServerNs := getNfsServerNamespace() p := &Provisioner{ @@ -74,10 +79,16 @@ func NewProvisioner(stopCh chan struct{}, kubeClient *clientset.Clientset) (*Pro Value: getDefaultNFSServerType(), }, }, - useClusterIP: menv.Truthy(ProvisionerNFSServerUseClusterIP), + useClusterIP: menv.Truthy(ProvisionerNFSServerUseClusterIP), + k8sNodeLister: listersv1.NewNodeLister(k8sNodeInformer.GetIndexer()), + nodeAffinity: getNodeAffinityRules(), } p.getVolumeConfig = p.GetVolumeConfig + // Running node informer will fetch node information from API Server + // and maintain it in cache + go k8sNodeInformer.Run(stopCh) + return p, nil } @@ -108,6 +119,14 @@ func (p *Provisioner) Provision(opts pvController.ProvisionOptions) (*v1.Persist return nil, err } + // Validate nodeAffinity rules for scheduling + // There might be changes to node after deploying + // NFS Provisioner + err = p.validateNodeAffinityRules() + if err != nil { + return nil, err + } + nfsServerType := pvCASConfig.GetNFSServerTypeFromConfig() size := resource.Quantity{} @@ -193,6 +212,32 @@ func (p *Provisioner) Delete(pv *v1.PersistentVolume) (err error) { return nil } +// validateNodeAffinityRules will returns error if there are no +// node exist for given affinity rules +func (p *Provisioner) validateNodeAffinityRules() error { + if len(p.nodeAffinity.MatchExpressions) == 0 { + return nil + } + + nodeSelector, err := v1helper.NodeSelectorRequirementsAsSelector(p.nodeAffinity.MatchExpressions) + if err != nil { + return err + } + + nodeList, err := p.k8sNodeLister.List(nodeSelector) + if err != nil { + return err + } + + if len(nodeList) == 0 { + return errors.Errorf( + "No matching nodes found for given affinity rules (%s)", + nodeSelector.String(), + ) + } + return nil +} + // sendEventOrIgnore sends anonymous nfs-pv provision/delete events func sendEventOrIgnore(pvcName, pvName, capacity, stgType, method string) { if !menv.Truthy(menv.OpenEBSEnableAnalytics) { diff --git a/provisioner/types.go b/provisioner/types.go index af61030..7049ce3 100644 --- a/provisioner/types.go +++ b/provisioner/types.go @@ -19,8 +19,9 @@ package provisioner import ( mconfig "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" + listerv1 "k8s.io/client-go/listers/core/v1" ) //Provisioner struct has the configuration and utilities required @@ -47,6 +48,12 @@ type Provisioner struct { //determine if clusterIP or clusterDNS should be used useClusterIP bool + + // k8sNodeLister hold cache information about nodes + k8sNodeLister listerv1.NodeLister + + // nodeAffinity specifies requirements for scheduling NFS Server + nodeAffinity NodeAffinity } //VolumeConfig struct contains the merged configuration of the PVC @@ -72,4 +79,63 @@ type VolumeConfig struct { // GetVolumeConfigFn allows to plugin a custom function // and makes it easy to unit test provisioner -type GetVolumeConfigFn func(pvName string, pvc *v1.PersistentVolumeClaim) (*VolumeConfig, error) +type GetVolumeConfigFn func(pvName string, pvc *corev1.PersistentVolumeClaim) (*VolumeConfig, error) + +// NodeAffinity represents group of node affinity scheduling +// rules that will be applied on NFS Server instance. If it is +// not configured then matches to no object i.e NFS Server can +// schedule on any node in a cluster. Configured values will be +// propogated to deployment.spec.template.spec.affinity.nodeAffinity. +// requiredDuringSchedulingIgnoredDuringExecution +// +// Values are propagated via ENV(NodeAffinity) on NFS Provisioner. +// Example: Following can be various options to specify NodeAffinity rules +// +// Config 1: Configure across zones and also storage should be available +// Env Value: "kubernetes.io/hostName:[z1-host1,z2-host1,z3-host1],kubernetes.io/storage:[available]" +// +// Config 1 will be propogated as shown below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/hostName +// operator: "In" +// values: +// - z1-host1 +// - z2-host2 +// - z3-host3 +// - key: kubernetes.io/storage +// operator: "In" +// values: +// - available +// +// Config2: Configure on storage nodes in zone1 +// Env Value: "kubernetes.io/storage:[],kubernetes.io/zone:[zone1]" +// +// Config2 will be propogated as shown below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/storage +// operator: "Exists" +// - key: kubernetes.io/zone +// operator: "In" +// values: +// - zone1 +// +// +// Configi3: Configure on any storage node +// Env Value: "kubernetes.io/storage:[]" +// +// Config3 will be propogated as below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/storage +// operator: "Exists" +// +// Like shown above various combinations can be specified and before +// provisioning configuration will be validated +// +// NOTE: All the comma separated specification will be ANDed +type NodeAffinity struct { + // A list of node selector requirements by node's labels + MatchExpressions []corev1.NodeSelectorRequirement +} diff --git a/tests/k8s_utils.go b/tests/k8s_utils.go index e79c48b..2ddd4e9 100644 --- a/tests/k8s_utils.go +++ b/tests/k8s_utils.go @@ -242,17 +242,18 @@ func (k *KubeClient) createDeployment(deployment *appsv1.Deployment) error { func (k *KubeClient) applyDeployment(deployment *appsv1.Deployment) error { // TODO: Use server side apply - _, err := k.AppsV1().Deployments(deployment.Namespace).Create(deployment) - if err != nil && !k8serrors.IsAlreadyExists(err) { - return errors.Errorf("Failed to create deployment %s/%s, err=%s", deployment.Namespace, deployment.Name, err) - } - if err == nil { - return nil - } - currentDeployment, err := k.AppsV1(). Deployments(deployment.Namespace). Get(deployment.Name, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + _, err := k.AppsV1().Deployments(deployment.Namespace).Create(deployment) + if err != nil { + return errors.Errorf("Failed to create deployment %s/%s, err=%s", deployment.Namespace, deployment.Name, err) + } + } + return err + } data, _, err := getPatchData(currentDeployment, deployment) if err != nil { @@ -285,6 +286,10 @@ func (k *KubeClient) updateDeployment(deployment *appsv1.Deployment) (*appsv1.De return k.AppsV1().Deployments(deployment.Namespace).Update(deployment) } +func (k *KubeClient) listDeployments(namespace, labelSelector string) (*appsv1.DeploymentList, error) { + return k.AppsV1().Deployments(namespace).List(metav1.ListOptions{LabelSelector: labelSelector}) +} + func dumpK8sObject(obj runtime.Object) { if encoder == nil { fmt.Printf("encoder not initilized\n") @@ -310,6 +315,11 @@ func (k *KubeClient) deleteStorageClass(scName string) error { return k.StorageV1().StorageClasses().Delete(scName, &metav1.DeleteOptions{}) } +// Add Node related operations +func (k *KubeClient) listNodes(labelSelector string) (*corev1.NodeList, error) { + return k.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: labelSelector}) +} + func getPatchData(oldObj, newObj interface{}) ([]byte, []byte, error) { oldData, err := json.Marshal(oldObj) if err != nil { diff --git a/tests/node_affinity_test.go b/tests/node_affinity_test.go new file mode 100644 index 0000000..fe9ef76 --- /dev/null +++ b/tests/node_affinity_test.go @@ -0,0 +1,224 @@ +/* +Copyright 2021 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + pvc "github.com/openebs/dynamic-nfs-provisioner/pkg/kubernetes/api/core/v1/persistentvolumeclaim" + provisioner "github.com/openebs/dynamic-nfs-provisioner/provisioner" + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("TEST NODE AFFINITY FEATURE", func() { + var ( + openebsNamespace = "openebs" + accessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + nodeAffinityKeys = []string{"kubernetes.io/hostname"} + capacity = "2Gi" + nfsProvisionerLabel = "openebs.io/component-name=openebs-nfs-provisioner" + nfsProvisionerContainerName = "openebs-provisioner-nfs" + pvcName = "node-affinity-pvc-nfs" + nodeAffinityKeyValues map[string][]string + ) + + When("node affinity environment variable is added", func() { + It("should be applied", func() { + nodeList, err := Client.listNodes("") + Expect(err).To(BeNil(), "failed to list nodes") + var nodeAffinityAsValue string + + nodeAffinityKeyValues = make(map[string][]string, len(nodeAffinityKeys)) + // Form affinity rules from multiple nodes + for _, node := range nodeList.Items { + for _, key := range nodeAffinityKeys { + if value, isExist := node.Labels[key]; isExist { + nodeAffinityKeyValues[key] = append(nodeAffinityKeyValues[key], value) + } + } + } + + for key, values := range nodeAffinityKeyValues { + nodeAffinityAsValue += key + ":[" + for _, value := range values { + nodeAffinityAsValue += value + "," + } + // remove extra comma + nodeAffinityAsValue = nodeAffinityAsValue[:len(nodeAffinityAsValue)-1] + nodeAffinityAsValue += "]," + } + // remove extra comma and add key as affinity rules + nodeAffinityAsValue = nodeAffinityAsValue[:len(nodeAffinityAsValue)-1] + ",kubernetes.io/arch" + nodeAffinityKeyValues["kubernetes.io/arch"] = []string{} + + deploymentList, err := Client.listDeployments(openebsNamespace, nfsProvisionerLabel) + Expect(err).To(BeNil(), "failed to list NFS Provisioner deployments") + + nfsProvisionerDeployment := deploymentList.Items[0] + for index, containerDetails := range nfsProvisionerDeployment.Spec.Template.Spec.Containers { + if containerDetails.Name == nfsProvisionerContainerName { + nfsProvisionerDeployment.Spec. + Template. + Spec. + Containers[index].Env = append( + nfsProvisionerDeployment.Spec. + Template. + Spec. + Containers[index].Env, + corev1.EnvVar{ + Name: string(provisioner.NodeAffinityKey), + Value: nodeAffinityAsValue, + }, + ) + break + } + } + + err = Client.applyDeployment(&nfsProvisionerDeployment) + Expect(err).To(BeNil(), "failed to add %s env to NFS Provisioner", provisioner.NodeAffinityKey) + }) + }) + + When("pvc with storageclass openebs-rwx is created", func() { + It("should create NFS Server with affinity rules", func() { + var ( + scName = "openebs-rwx" + ) + + By("building a pvc") + pvcObj, err := pvc.NewBuilder(). + WithName(pvcName). + WithNamespace(applicationNamespace). + WithStorageClass(scName). + WithAccessModes(accessModes). + WithCapacity(capacity).Build() + Expect(err).ShouldNot( + HaveOccurred(), + "while building pvc {%s} in namespace {%s}", + pvcName, + applicationNamespace, + ) + + By("creating above pvc") + err = Client.createPVC(pvcObj) + Expect(err).To( + BeNil(), + "while creating pvc {%s} in namespace {%s}", + pvcName, + applicationNamespace, + ) + + _, err = Client.waitForPVCBound(pvcObj.Name, pvcObj.Namespace) + Expect(err).To(BeNil(), "While waiting for PVC to get into bound state") + + boundedPVCObj, err := Client.getPVC(pvcObj.Namespace, pvcObj.Name) + Expect(err).To(BeNil(), "While fetching bounded PVC") + + nfsServerLabel := "openebs.io/nfs-server=nfs-" + boundedPVCObj.Spec.VolumeName + err = Client.waitForPods(openebsNamespace, nfsServerLabel, corev1.PodRunning, 1) + Expect(err).To(BeNil(), "while verifying pod count") + + // Get NFS Server deployment + nfsServerDeployment, err := Client.getDeployment(openebsNamespace, "nfs-"+boundedPVCObj.Spec.VolumeName) + Expect(err).To(BeNil(), "failed to list NFS Provisioner deployments") + + Expect(nfsServerDeployment.Spec.Template.Spec.Affinity).NotTo( + BeNil(), + "affinity should exist on %s/%s NFS Server deployment", + nfsServerDeployment.Namespace, + nfsServerDeployment.Name, + ) + Expect(nfsServerDeployment.Spec.Template.Spec.Affinity.NodeAffinity).NotTo( + BeNil(), + "node affinity should exist on %s/%s NFS Server deployment", + nfsServerDeployment.Namespace, + nfsServerDeployment.Name, + ) + Expect(nfsServerDeployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).NotTo( + BeNil(), + "requiredDuringSchedulingIgnoreDuringExecution should exist on %s/%s NFS Server deployment", + nfsServerDeployment.Namespace, + nfsServerDeployment.Name, + ) + + // Verify propogation of affinity rules + for _, rules := range nfsServerDeployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + for _, affinityRule := range rules.MatchExpressions { + values, isExist := nodeAffinityKeyValues[affinityRule.Key] + Expect(isExist).Should(BeTrue(), "unknown key %s added under node affinity rules", affinityRule.Key) + if len(values) == 0 { + Expect(affinityRule.Operator).Should( + Equal(corev1.NodeSelectorOpExists), + "operator for key %s should be %s", + affinityRule.Key, + corev1.NodeSelectorOpExists, + ) + Expect(affinityRule.Values).Should(BeNil(), "values should not exist") + } else { + Expect(affinityRule.Operator).Should( + Equal(corev1.NodeSelectorOpIn), + "operator for key %s should be %s", + affinityRule.Key, + corev1.NodeSelectorOpIn, + ) + Expect(affinityRule.Values).Should(Equal(values), "values should match with affinity rules") + } + } + } + }) + }) + + When("node affinty rules are removed from env", func() { + It("should remove from the NFS provisioner", func() { + deploymentList, err := Client.listDeployments(openebsNamespace, nfsProvisionerLabel) + Expect(err).To(BeNil(), "failed to list NFS Provisioner deployments") + + nfsProvisionerDeployment := deploymentList.Items[0] + for cIndex, containerDetails := range nfsProvisionerDeployment.Spec.Template.Spec.Containers { + if containerDetails.Name == nfsProvisionerContainerName { + envIndex := 0 + for _, envVar := range containerDetails.Env { + if envVar.Name == string(provisioner.NodeAffinityKey) { + break + } + envIndex++ + } + nfsProvisionerDeployment.Spec.Template.Spec.Containers[cIndex].Env = append(nfsProvisionerDeployment.Spec.Template.Spec.Containers[cIndex].Env[:envIndex], nfsProvisionerDeployment.Spec.Template.Spec.Containers[cIndex].Env[envIndex+1:]...) + break + } + } + + err = Client.applyDeployment(&nfsProvisionerDeployment) + Expect(err).To(BeNil(), "failed to add %s env to NFS Provisioner", provisioner.NodeAffinityKey) + }) + }) + + When("pvc with storageclass openebs-rwx is deleted ", func() { + It("should delete the pvc", func() { + + By("deleting above pvc") + err = Client.deletePVC(applicationNamespace, pvcName) + Expect(err).To( + BeNil(), + "while deleting pvc {%s} in namespace {%s}", + pvcName, + applicationNamespace, + ) + + }) + }) +})