From 4ca016606ff1118cf2be39a5c429ea9e97745f8e Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 5 Aug 2022 15:33:12 -0400 Subject: [PATCH] draft --- Makefile | 34 +++- tests/e2e/backup_restore_suite_test.go | 18 ++ tests/e2e/lib/operators.go | 240 +++++++++++++++++++++++++ tests/e2e/volsync/volsync-sub.yaml | 16 ++ 4 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/lib/operators.go create mode 100644 tests/e2e/volsync/volsync-sub.yaml diff --git a/Makefile b/Makefile index 7b896d37248..a4924a72ff9 100644 --- a/Makefile +++ b/Makefile @@ -330,7 +330,7 @@ deploy-olm: IMG=$(THIS_OPERATOR_IMAGE) BUNDLE_IMG=$(THIS_BUNDLE_IMAGE) \ make docker-build docker-push bundle bundle-build bundle-push; \ rm -rf $(DEPLOY_TMP) - operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE) + operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE) --index-image=quay.io/operator-framework/opm:v1.23.0 .PHONY: opm OPM = ./bin/opm @@ -384,6 +384,7 @@ sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//")) || $(CLUSTER_TY #TEST_FILTER := $(shell echo '! aws && ! gcp && ! azure' | sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//") SETTINGS_TMP=/tmp/test-settings +.PHONY: test-e2e-setup test-e2e-setup: mkdir -p $(SETTINGS_TMP) TARGET_CI_CRED_FILE="$(CI_CRED_FILE)" AZURE_RESOURCE_FILE="$(AZURE_RESOURCE_FILE)" CI_JSON_CRED_FILE="$(AZURE_CI_JSON_CRED_FILE)" \ @@ -391,7 +392,8 @@ test-e2e-setup: PROVIDER="$(VELERO_PLUGIN)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \ VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh" -test-e2e: test-e2e-setup +.PHONY: test-e2e-ginkgo +test-e2e-ginkgo: test-e2e-setup ginkgo run -mod=mod tests/e2e/ -- -credentials=$(OADP_CRED_FILE) \ -velero_namespace=$(OADP_TEST_NAMESPACE) \ -settings=$(SETTINGS_TMP)/oadpcreds \ @@ -404,5 +406,31 @@ test-e2e: test-e2e-setup -artifact_dir=$(ARTIFACT_DIR) \ -oc_cli=$(OC_CLI) -test-e2e-cleanup: +.PHONY: test-e2e +test-e2e: volsync-install test-e2e-ginkgo + +.PHONY: test-e2e-cleanup +test-e2e-cleanup: volsync-uninstall rm -rf $(SETTINGS_TMP) + +.PHONY: volsync-install +volsync-install: + $(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}')) + # OperatorGroup not required, volsync is global operator which has operatorgroup already. + # Create subscription for operator if not installed. + @if [ "$(VS_CURRENT_CSV)" == "" ]; then \ + $(OC_CLI) replace --force -f tests/e2e/volsync/volsync-sub.yaml; \ + else \ + echo $(VS_CURRENT_CSV) already installed; \ + fi + +.PHONY: volsync-uninstall +volsync-uninstall: + $(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}')) + @if [ "$(VS_CURRENT_CSV)" != "" ]; then \ + echo "Uninstalling $(VS_CURRENT_CSV)"; \ + $(OC_CLI) delete subscription volsync-product -n openshift-operators && \ + $(OC_CLI) delete csv $(VS_CURRENT_CSV) -n openshift-operators; \ + else \ + echo No subscription found, skipping uninstall; \ + fi \ No newline at end of file diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 1717d227ea5..5bb72879ee3 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/openshift/oadp-operator/pkg/common" . "github.com/openshift/oadp-operator/tests/e2e/lib" utils "github.com/openshift/oadp-operator/tests/e2e/utils" "sigs.k8s.io/controller-runtime/pkg/client" @@ -16,6 +17,23 @@ import ( type VerificationFunction func(client.Client, string) error +func dataMoverReady(preBackupState bool, backupRestoreType BackupRestoreType, preVerificationFunction VerificationFunction) VerificationFunction { + return VerificationFunction(func(ocClient client.Client, namespace string) error { + // install volsync controller + + + // run pre-verification function + err := preVerificationFunction(ocClient, namespace) + if err != nil { + return err + } + + // check volsync controller is ready + fmt.Printf("waiting for volsync controller readiness") + Eventually(IsDeploymentReady(ocClient, common.VolSyncDeploymentNamespace, common.VolSyncDeploymentName), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + }) +} + func mongoready(preBackupState bool, backupRestoreType BackupRestoreType) VerificationFunction { return VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(IsDCReady(ocClient, namespace, "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) diff --git a/tests/e2e/lib/operators.go b/tests/e2e/lib/operators.go new file mode 100644 index 00000000000..0b7122c8a94 --- /dev/null +++ b/tests/e2e/lib/operators.go @@ -0,0 +1,240 @@ +package lib + +import "sigs.k8s.io/controller-runtime/pkg/client" + +// install Operator +func InstallOperator(ocClient client.Client) error { + // View list of operators available to the cluster from operatorhub + // ❯ oc get packagemanifests | grep volsync + // volsync-product Red Hat Operators 102d +// ❯ oc describe packagemanifests volsync-product Red Hat Operators 102d +// Name: volsync-product +// Namespace: openshift-storage +// Labels: catalog=redhat-operators +// catalog-namespace=openshift-marketplace +// operatorframework.io/arch.amd64=supported +// operatorframework.io/arch.arm64=supported +// operatorframework.io/arch.ppc64le=supported +// operatorframework.io/arch.s390x=supported +// operatorframework.io/os.linux=supported +// provider=Red Hat +// provider-url= +// Annotations: +// API Version: packages.operators.coreos.com/v1 +// Kind: PackageManifest +// Metadata: +// Creation Timestamp: 2022-04-25T15:01:29Z +// Spec: +// Status: +// Catalog Source: redhat-operators +// Catalog Source Display Name: Red Hat Operators +// Catalog Source Namespace: openshift-marketplace +// Catalog Source Publisher: Red Hat +// Channels: +// Current CSV: volsync-product.v0.4.1 +// Current CSV Desc: +// Annotations: +// Alm - Examples: [ +// { +// "apiVersion": "volsync.backube/v1alpha1", +// "kind": "ReplicationDestination", +// "metadata": { +// "name": "replicationdestination-sample" +// }, +// "spec": { +// "rsync": { +// "accessModes": [ +// "ReadWriteOnce" +// ], +// "capacity": "10Gi", +// "copyMethod": "Snapshot", +// "serviceType": "ClusterIP" +// } +// } +// }, +// { +// "apiVersion": "volsync.backube/v1alpha1", +// "kind": "ReplicationSource", +// "metadata": { +// "name": "replicationsource-sample" +// }, +// "spec": { +// "rsync": { +// "address": "my.host.com", +// "copyMethod": "Clone", +// "sshKeys": "secretRef" +// }, +// "sourcePVC": "pvcname", +// "trigger": { +// "schedule": "0 * * * *" +// } +// } +// } +// ] +// Capabilities: Basic Install +// Container Image: registry.redhat.io/rhacm2/volsync-rhel8@sha256:df630925f29753454f0d7de74ac1f2c751921bd73f4c4defaefe4b83154e494c +// Created At: 01 Jul 2022, 18:29 +// operators.openshift.io/infrastructure-features: ["disconnected", "fips"] +// operators.openshift.io/valid-subscription: ["OpenShift Platform Plus", "Red Hat Advanced Cluster Management for Kubernetes"] +// operators.operatorframework.io/builder: operator-sdk-v1.14.0+git +// operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 +// Repository: https://github.com/backube/volsync +// Support: Red Hat +// Apiservicedefinitions: +// Customresourcedefinitions: +// Owned: +// Description: ReplicationDestination defines the destination for a replicated volume +// Display Name: Replication Destination +// Kind: ReplicationDestination +// Name: replicationdestinations.volsync.backube +// Version: v1alpha1 +// Description: ReplicationSource defines the source for a replicated volume +// Display Name: Replication Source +// Kind: ReplicationSource +// Name: replicationsources.volsync.backube +// Version: v1alpha1 +// Description: Asynchronous volume replication for Kubernetes CSI storage +// Display Name: VolSync +// Install Modes: +// Supported: false +// Type: OwnNamespace +// Supported: false +// Type: SingleNamespace +// Supported: false +// Type: MultiNamespace +// Supported: true +// Type: AllNamespaces +// Keywords: +// backup +// csi +// data +// disaster recovery +// replication +// storage +// Links: +// Name: Documentation +// URL: https://access.redhat.com/documentation/en-us/red_hat_advanced_cluster_management_for_kubernetes/2.5/html-single/add-ons/index#volsync +// Name: Source +// URL: https://github.com/backube/volsync +// Maintainers: +// Email: acm-contact@redhat.com +// Name: Red Hat ACM Team +// Maturity: alpha +// Min Kube Version: 1.20.0 +// Provider: +// Name: Red Hat +// Related Images: +// registry.redhat.io/openshift4/ose-kube-rbac-proxy@sha256:b6e06bf125c2c919f4a74098d9bf9bec05122b421213dd6d442de800f5ef3cf7 +// registry.redhat.io/rhacm2/volsync-rhel8@sha256:df630925f29753454f0d7de74ac1f2c751921bd73f4c4defaefe4b83154e494c +// registry.redhat.io/rhacm2/volsync-mover-rsync-rhel8@sha256:e575a16d023af99f0a4f86b1f220c0abaae1e8973e60cb770df82eb5da362ca2 +// registry.redhat.io/rhacm2/volsync-mover-rclone-rhel8@sha256:e39f9f36b0ff65bff5f151d1200b5e855ac1972661c4e531c5138111baac608b +// registry.redhat.io/rhacm2/volsync-mover-restic-rhel8@sha256:37cf9057f86d7b9f5d1dc44ac16f5cba274527391cffcc6d012af2c0bd0ee54b +// Version: 0.4.1 +// Name: acm-2.5 +// Current CSV: volsync-product.v0.4.1 +// Current CSV Desc: +// Annotations: +// Alm - Examples: [ +// { +// "apiVersion": "volsync.backube/v1alpha1", +// "kind": "ReplicationDestination", +// "metadata": { +// "name": "replicationdestination-sample" +// }, +// "spec": { +// "rsync": { +// "accessModes": [ +// "ReadWriteOnce" +// ], +// "capacity": "10Gi", +// "copyMethod": "Snapshot", +// "serviceType": "ClusterIP" +// } +// } +// }, +// { +// "apiVersion": "volsync.backube/v1alpha1", +// "kind": "ReplicationSource", +// "metadata": { +// "name": "replicationsource-sample" +// }, +// "spec": { +// "rsync": { +// "address": "my.host.com", +// "copyMethod": "Clone", +// "sshKeys": "secretRef" +// }, +// "sourcePVC": "pvcname", +// "trigger": { +// "schedule": "0 * * * *" +// } +// } +// } +// ] +// Capabilities: Basic Install +// Container Image: registry.redhat.io/rhacm2/volsync-rhel8@sha256:df630925f29753454f0d7de74ac1f2c751921bd73f4c4defaefe4b83154e494c +// Created At: 01 Jul 2022, 18:29 +// operators.openshift.io/infrastructure-features: ["disconnected", "fips"] +// operators.openshift.io/valid-subscription: ["OpenShift Platform Plus", "Red Hat Advanced Cluster Management for Kubernetes"] +// operators.operatorframework.io/builder: operator-sdk-v1.14.0+git +// operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 +// Repository: https://github.com/backube/volsync +// Support: Red Hat +// Apiservicedefinitions: +// Customresourcedefinitions: +// Owned: +// Description: ReplicationDestination defines the destination for a replicated volume +// Display Name: Replication Destination +// Kind: ReplicationDestination +// Name: replicationdestinations.volsync.backube +// Version: v1alpha1 +// Description: ReplicationSource defines the source for a replicated volume +// Display Name: Replication Source +// Kind: ReplicationSource +// Name: replicationsources.volsync.backube +// Version: v1alpha1 +// Description: Asynchronous volume replication for Kubernetes CSI storage +// Display Name: VolSync +// Install Modes: +// Supported: false +// Type: OwnNamespace +// Supported: false +// Type: SingleNamespace +// Supported: false +// Type: MultiNamespace +// Supported: true +// Type: AllNamespaces +// Keywords: +// backup +// csi +// data +// disaster recovery +// replication +// storage +// Links: +// Name: Documentation +// URL: https://access.redhat.com/documentation/en-us/red_hat_advanced_cluster_management_for_kubernetes/2.5/html-single/add-ons/index#volsync +// Name: Source +// URL: https://github.com/backube/volsync +// Maintainers: +// Email: acm-contact@redhat.com +// Name: Red Hat ACM Team +// Maturity: alpha +// Min Kube Version: 1.20.0 +// Provider: +// Name: Red Hat +// Related Images: +// registry.redhat.io/openshift4/ose-kube-rbac-proxy@sha256:b6e06bf125c2c919f4a74098d9bf9bec05122b421213dd6d442de800f5ef3cf7 +// registry.redhat.io/rhacm2/volsync-rhel8@sha256:df630925f29753454f0d7de74ac1f2c751921bd73f4c4defaefe4b83154e494c +// registry.redhat.io/rhacm2/volsync-mover-rclone-rhel8@sha256:e39f9f36b0ff65bff5f151d1200b5e855ac1972661c4e531c5138111baac608b +// registry.redhat.io/rhacm2/volsync-mover-restic-rhel8@sha256:37cf9057f86d7b9f5d1dc44ac16f5cba274527391cffcc6d012af2c0bd0ee54b +// registry.redhat.io/rhacm2/volsync-mover-rsync-rhel8@sha256:e575a16d023af99f0a4f86b1f220c0abaae1e8973e60cb770df82eb5da362ca2 +// Version: 0.4.1 +// Name: stable +// Default Channel: stable +// Package Name: volsync-product +// Provider: +// Name: Red Hat + + return nil +} \ No newline at end of file diff --git a/tests/e2e/volsync/volsync-sub.yaml b/tests/e2e/volsync/volsync-sub.yaml new file mode 100644 index 00000000000..3f5f0eb8469 --- /dev/null +++ b/tests/e2e/volsync/volsync-sub.yaml @@ -0,0 +1,16 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + labels: + operators.coreos.com/volsync-product.openshift-operators: "" + name: volsync-product + namespace: openshift-operators +spec: + channel: stable + installPlanApproval: Automatic + name: volsync-product + source: redhat-operators + sourceNamespace: openshift-marketplace + # v0.4.1 was latest version of the operator. We comment this out so it always install the latest version. + # If it breaks in the future we can re-specify startingCSV to install a specific version. + # startingCSV: volsync-product.v0.4.1