Add data mover to e2e

Signed-off-by: Tiger Kaovilai <[email protected]>

Makefile

@@ -330,7 +330,7 @@ deploy-olm:
 	IMG=$(THIS_OPERATOR_IMAGE) BUNDLE_IMG=$(THIS_BUNDLE_IMAGE) \
 		make docker-build docker-push bundle bundle-build bundle-push; \
 	rm -rf $(DEPLOY_TMP)
-	operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE)
+	operator-sdk run bundle $(THIS_BUNDLE_IMAGE) --namespace $(OADP_TEST_NAMESPACE) --index-image=quay.io/operator-framework/opm:v1.23.0
 
 .PHONY: opm
 OPM = ./bin/opm
@@ -384,14 +384,16 @@ sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//")) || $(CLUSTER_TY
 #TEST_FILTER := $(shell echo '! aws && ! gcp && ! azure' | sed -r "s/[&]* [!] $(CLUSTER_TYPE)|[!] $(CLUSTER_TYPE) [&]*//")
 SETTINGS_TMP=/tmp/test-settings
 
+.PHONY: test-e2e-setup
 test-e2e-setup:
 	mkdir -p $(SETTINGS_TMP)
 	TARGET_CI_CRED_FILE="$(CI_CRED_FILE)" AZURE_RESOURCE_FILE="$(AZURE_RESOURCE_FILE)" CI_JSON_CRED_FILE="$(AZURE_CI_JSON_CRED_FILE)" \
 	OADP_JSON_CRED_FILE="$(AZURE_OADP_JSON_CRED_FILE)" OADP_CRED_FILE="$(OADP_CRED_FILE)" OPENSHIFT_CI="$(OPENSHIFT_CI)" \
 	PROVIDER="$(VELERO_PLUGIN)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \
 	VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh"
 
-test-e2e: test-e2e-setup
+.PHONY: test-e2e-ginkgo
+test-e2e-ginkgo: test-e2e-setup
 	ginkgo run -mod=mod tests/e2e/ -- -credentials=$(OADP_CRED_FILE) \
 	-velero_namespace=$(OADP_TEST_NAMESPACE) \
 	-settings=$(SETTINGS_TMP)/oadpcreds \
@@ -404,5 +406,31 @@ test-e2e: test-e2e-setup
 	-artifact_dir=$(ARTIFACT_DIR) \
 	-oc_cli=$(OC_CLI)
 
-test-e2e-cleanup:
+.PHONY: test-e2e
+test-e2e: volsync-install test-e2e-ginkgo
+
+.PHONY: test-e2e-cleanup
+test-e2e-cleanup: volsync-uninstall
 	rm -rf $(SETTINGS_TMP)
+
+.PHONY: volsync-install
+volsync-install:
+	$(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}'))
+	# OperatorGroup not required, volsync is global operator which has operatorgroup already.
+	# Create subscription for operator if not installed.
+	@if [ "$(VS_CURRENT_CSV)" == "" ]; then \
+		$(OC_CLI) replace --force -f tests/e2e/volsync/volsync-sub.yaml; \
+	else \
+		echo $(VS_CURRENT_CSV) already installed; \
+	fi
+
+.PHONY: volsync-uninstall
+volsync-uninstall:
+	$(eval VS_CURRENT_CSV:=$(shell oc get subscription volsync-product -n openshift-operators -ojsonpath='{.status.currentCSV}'))
+	@if [ "$(VS_CURRENT_CSV)" != "" ]; then \
+		echo "Uninstalling $(VS_CURRENT_CSV)"; \
+		$(OC_CLI) delete subscription volsync-product -n openshift-operators && \
+		$(OC_CLI) delete csv $(VS_CURRENT_CSV) -n openshift-operators; \
+	else \
+		echo No subscription found, skipping uninstall; \
+	fi

go.mod

@@ -24,6 +24,8 @@ require (
 	sigs.k8s.io/controller-runtime v0.11.0
 )
 
+require github.com/google/go-cmp v0.5.6
+
 require (
 	cloud.google.com/go v0.93.3 // indirect
 	github.com/Azure/azure-sdk-for-go v61.4.0+incompatible // indirect
@@ -52,7 +54,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/hashicorp/go-hclog v0.14.1 // indirect

tests/e2e/backup_restore_suite_test.go

@@ -11,6 +11,8 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	"github.com/onsi/ginkgo/v2/types"
 	. "github.com/onsi/gomega"
+	"github.com/openshift/oadp-operator/controllers"
+	"github.com/openshift/oadp-operator/pkg/common"
 	. "github.com/openshift/oadp-operator/tests/e2e/lib"
 	corev1 "k8s.io/api/core/v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
@@ -20,6 +22,19 @@ import (
 
 type VerificationFunction func(client.Client, string) error
 
+type appVerificationFunction func(bool, BackupRestoreType) VerificationFunction
+
+func dataMoverReady(preBackupState bool, appVerificationFunction appVerificationFunction) VerificationFunction {
+	return VerificationFunction(func(ocClient client.Client, namespace string) error {
+		// check volsync subscription exists
+		Eventually(InstalledSubscriptionCSV(ocClient, "openshift-operators", "volsync-product"), timeoutMultiplier*time.Minute*10, time.Second*10).ShouldNot(Equal(""))
+		// check volsync controller is ready
+		fmt.Printf("waiting for volsync controller readiness")
+		Eventually(IsDeploymentReady(ocClient, common.VolSyncDeploymentNamespace, common.VolSyncDeploymentName), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
+		return appVerificationFunction(preBackupState, CSIDataMover)(ocClient, namespace)
+	})
+}
+
 func mongoready(preBackupState bool, backupRestoreType BackupRestoreType) VerificationFunction {
 	return VerificationFunction(func(ocClient client.Client, namespace string) error {
 		Eventually(IsDCReady(ocClient, namespace, "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
@@ -79,15 +94,17 @@ var _ = Describe("AWS backup restore tests", func() {
 			GinkgoWriter.Println("End of velero deployment pod logs")
 		}
 		// remove app namespace if leftover (likely previously failed before reaching uninstall applications) to clear items such as PVCs which are immutable so that next test can create new ones
-		err := dpaCR.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: v1.ObjectMeta{
-			Name: lastInstallingApplicationNamespace,
-			Namespace: lastInstallingApplicationNamespace,
-		}}, &client.DeleteOptions{})
-		if k8serror.IsNotFound(err) {
-			err = nil
+		if lastInstallingApplicationNamespace != dpaCR.Namespace {
+			err := dpaCR.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: v1.ObjectMeta{
+				Name: lastInstallingApplicationNamespace,
+				Namespace: lastInstallingApplicationNamespace,
+			}}, &client.DeleteOptions{})
+			if k8serror.IsNotFound(err) {
+				err = nil
+			}
+			Expect(err).ToNot(HaveOccurred())
 		}
-		Expect(err).ToNot(HaveOccurred())
-		err = dpaCR.Delete()
+		err := dpaCR.Delete()
 		Expect(err).ToNot(HaveOccurred())
 	})
 
@@ -139,14 +156,24 @@ var _ = Describe("AWS backup restore tests", func() {
 				log.Printf("Waiting for restic pods to be running")
 				Eventually(AreResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
 			}
-			if brCase.BackupRestoreType == CSI {
+			if brCase.BackupRestoreType == CSI || brCase.BackupRestoreType == CSIDataMover {
 				if provider == "aws" || provider == "ibmcloud" || provider == "gcp" || provider == "azure" {
 					log.Printf("Creating VolumeSnapshotClass for CSI backuprestore of %s", brCase.Name)
 					snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider)
 					err = InstallApplication(dpaCR.Client, snapshotClassPath)
 					Expect(err).ToNot(HaveOccurred())
 				}
-
+				if brCase.BackupRestoreType == CSIDataMover {
+					dpaCR.Client.Create(context.Background(), &corev1.Secret{
+						ObjectMeta: v1.ObjectMeta{
+							Name:      controllers.ResticsecretName,
+							Namespace: dpaCR.Namespace,
+						},
+						StringData: map[string]string{
+							controllers.ResticPassword: "e2e-restic-password",
+						},
+					}, &client.CreateOptions{})
+				}
 			}
 
 			// TODO: check registry deployments are deleted
@@ -256,15 +283,15 @@ var _ = Describe("AWS backup restore tests", func() {
 
 		},
 		Entry("MySQL application CSI", Label("ibmcloud", "aws", "gcp", "azure"), BackupRestoreCase{
-			ApplicationTemplate:  fmt.Sprintf("./sample-applications/mysql-persistent/mysql-persistent-csi.yaml"),
+			ApplicationTemplate:  "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml",
 			ApplicationNamespace: "mysql-persistent",
 			Name:                 "mysql-csi-e2e",
 			BackupRestoreType:    CSI,
 			PreBackupVerify:      mysqlReady(true, CSI),
 			PostRestoreVerify:    mysqlReady(false, CSI),
 		}, nil),
 		Entry("Mongo application CSI", Label("ibmcloud", "aws", "gcp", "azure"), BackupRestoreCase{
-			ApplicationTemplate:  fmt.Sprintf("./sample-applications/mongo-persistent/mongo-persistent-csi.yaml"),
+			ApplicationTemplate:  "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml",
 			ApplicationNamespace: "mongo-persistent",
 			Name:                 "mongo-csi-e2e",
 			BackupRestoreType:    CSI,
@@ -287,5 +314,21 @@ var _ = Describe("AWS backup restore tests", func() {
 			PreBackupVerify:      mysqlReady(true, RESTIC),
 			PostRestoreVerify:    mysqlReady(false, RESTIC),
 		}, nil),
+		Entry("Mongo application DATAMOVER", BackupRestoreCase{
+			ApplicationTemplate:  "./sample-applications/mongo-persistent/mongo-persistent.yaml",
+			ApplicationNamespace: "mongo-persistent",
+			Name:                 "mongo-restic-e2e",
+			BackupRestoreType:    CSIDataMover,
+			PreBackupVerify:      dataMoverReady(true, mongoready),
+			PostRestoreVerify:    dataMoverReady(false, mongoready),
+		}, nil),
+		Entry("MySQL application DATAMOVER", BackupRestoreCase{
+			ApplicationTemplate:  "./sample-applications/mysql-persistent/mysql-persistent.yaml",
+			ApplicationNamespace: "mysql-persistent",
+			Name:                 "mysql-restic-e2e",
+			BackupRestoreType:    CSIDataMover,
+			PreBackupVerify:      dataMoverReady(true, mysqlReady),
+			PostRestoreVerify:    dataMoverReady(false, mysqlReady),
+		}, nil),
 	)
 })

tests/e2e/lib/apps.go

@@ -24,6 +24,7 @@ import (
 	routev1 "github.com/openshift/api/route/v1"
 	security "github.com/openshift/api/security/v1"
 	templatev1 "github.com/openshift/api/template/v1"
+	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -347,6 +348,25 @@ func AreApplicationPodsRunning(namespace string) wait.ConditionFunc {
 	}
 }
 
+func InstalledSubscriptionCSV(ocClient client.Client, namespace, subscriptionName string) func() (string, error) {
+	return func() (string, error) {
+		// get operator-sdk subscription
+		subscription := &operatorsv1alpha1.Subscription{}
+		err := ocClient.Get(context.Background(), client.ObjectKey{
+			Namespace: namespace,
+			Name:      subscriptionName,
+		}, subscription)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return "", nil
+			}
+			ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Error getting subscription: %v\n", err)))
+			return "", err
+		}
+		return subscription.Status.InstalledCSV, nil
+	}
+}
+
 func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) {
 	log.Println("Printing events for namespace: ", namespace)
 	clientset, err := setUpClient()
@@ -454,7 +474,7 @@ func VerifyBackupRestoreData(artifact_dir string, namespace string, routeName st
 		return err
 	}
 	//Verifying backup-restore data only for CSI as of now.
-	if backupRestoretype == CSI || backupRestoretype == RESTIC {
+	if backupRestoretype == CSI || backupRestoretype == CSIDataMover || backupRestoretype == RESTIC {
 		//check if backupfile exists. If true { compare data response with data from file} (post restore step)
 		//else write data to backup-data.txt (prebackup step)
 		if _, err := os.Stat(backupFile); err == nil {

tests/e2e/lib/dpa_helpers.go

@@ -24,6 +24,7 @@ import (
 	oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1"
 	utils "github.com/openshift/oadp-operator/tests/e2e/utils"
 	operators "github.com/operator-framework/api/pkg/operators/v1alpha1"
+	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -39,6 +40,7 @@ type BackupRestoreType string
 
 const (
 	CSI    BackupRestoreType = "csi"
+	CSIDataMover    BackupRestoreType = "csi-datamover"
 	RESTIC BackupRestoreType = "restic"
 )
 
@@ -57,6 +59,10 @@ var Dpa *oadpv1alpha1.DataProtectionApplication
 func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
 	// Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins.
 	dpaInstance := oadpv1alpha1.DataProtectionApplication{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DataProtectionApplication",
+			APIVersion: "oadp.openshift.io/v1alpha1",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      v.Name,
 			Namespace: v.Namespace,
@@ -102,26 +108,32 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
 	for _, plugin := range dpaInstance.Spec.Configuration.Velero.DefaultPlugins {
 		defaultPlugins[plugin] = emptyStruct{}
 	}
-	featureFlags := make(map[string]emptyStruct)
+	veleroFeatureFlags := make(map[string]emptyStruct)
 	for _, flag := range dpaInstance.Spec.Configuration.Velero.FeatureFlags {
-		featureFlags[flag] = emptyStruct{}
+		veleroFeatureFlags[flag] = emptyStruct{}
 	}
+	dpaInstance.Spec.Features = &oadpv1alpha1.Features{DataMover: &oadpv1alpha1.DataMover{Enable: false}}
 	switch backupRestoreType {
 	case RESTIC:
 		dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(true)
 		delete(defaultPlugins, oadpv1alpha1.DefaultPluginCSI)
-		delete(featureFlags, "EnableCSI")
+		delete(veleroFeatureFlags, "EnableCSI")
 	case CSI:
 		dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(false)
 		defaultPlugins[oadpv1alpha1.DefaultPluginCSI] = emptyStruct{}
-		featureFlags["EnableCSI"] = emptyStruct{}
+		veleroFeatureFlags["EnableCSI"] = emptyStruct{}
+	case CSIDataMover:
+		dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(false)
+		defaultPlugins[oadpv1alpha1.DefaultPluginCSI] = emptyStruct{}
+		veleroFeatureFlags["EnableCSI"] = emptyStruct{}
+		dpaInstance.Spec.Features.DataMover.Enable = true
 	}
 	dpaInstance.Spec.Configuration.Velero.DefaultPlugins = make([]oadpv1alpha1.DefaultPlugin, 0)
 	for k := range defaultPlugins {
 		dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, k)
 	}
 	dpaInstance.Spec.Configuration.Velero.FeatureFlags = make([]string, 0)
-	for k := range featureFlags {
+	for k := range veleroFeatureFlags {
 		dpaInstance.Spec.Configuration.Velero.FeatureFlags = append(dpaInstance.Spec.Configuration.Velero.FeatureFlags, k)
 	}
 	v.CustomResource = &dpaInstance
@@ -173,14 +185,25 @@ func (v *DpaCustomResource) CreateOrUpdateWithRetries(spec *oadpv1alpha1.DataPro
 	)
 	for i := 0; i < retries; i++ {
 		if cr, err = v.Get(); apierrors.IsNotFound(err) {
-			v.Build(v.backupRestoreType)
-			v.CustomResource.Spec = *spec
+			v.CustomResource = &oadpv1alpha1.DataProtectionApplication{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "DataProtectionApplication",
+					APIVersion: "oadp.openshift.io/v1alpha1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      v.Name,
+					Namespace: v.Namespace,
+				},
+				Spec: *spec.DeepCopy(),
+			}
 			return v.Create()
 		} else if err != nil {
 			return err
 		}
-		cr.Spec = *spec
-		if err = v.Client.Update(context.Background(), cr); err != nil {
+		spec.DeepCopyInto(&cr.Spec)
+		cr.ObjectMeta.ManagedFields = nil
+		if err = v.Client.Patch(context.Background(), cr, client.Merge, &client.PatchOptions{}); err != nil {
+			log.Println("error patching velero cr", err)
 			if apierrors.IsConflict(err) && i < retries-1 {
 				log.Println("conflict detected during DPA CreateOrUpdate, retrying for ", retries-i-1, " more times")
 				time.Sleep(time.Second * 2)
@@ -219,6 +242,7 @@ func (v *DpaCustomResource) SetClient() error {
 	operators.AddToScheme(client.Scheme())
 	volumesnapshotv1.AddToScheme(client.Scheme())
 	buildv1.AddToScheme(client.Scheme())
+	operatorsv1alpha1.AddToScheme(client.Scheme())
 
 	v.Client = client
 	return nil

tests/e2e/volsync/volsync-sub.yaml

@@ -0,0 +1,16 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  labels:
+    operators.coreos.com/volsync-product.openshift-operators: ""
+  name: volsync-product
+  namespace: openshift-operators
+spec:
+  channel: stable
+  installPlanApproval: Automatic
+  name: volsync-product
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace
+  # v0.4.1 was latest version of the operator. We comment this out so it always install the latest version.
+  # If it breaks in the future we can re-specify startingCSV to install a specific version.
+  # startingCSV: volsync-product.v0.4.1