diff --git a/deploy/crds/noobaa.io_noobaas.yaml b/deploy/crds/noobaa.io_noobaas.yaml index 639a84454..babdca7e5 100644 --- a/deploy/crds/noobaa.io_noobaas.yaml +++ b/deploy/crds/noobaa.io_noobaas.yaml @@ -1911,6 +1911,10 @@ spec: description: Phase is a simple, high-level summary of where the System is in its lifecycle type: string + postgresMajorVersion: + description: PostgresMajorVersion is the major version of postgress + db image + type: string postgresUpdatePhase: description: Upgrade reports the status of the ongoing postgres upgrade process diff --git a/pkg/apis/noobaa/v1alpha1/noobaa_types.go b/pkg/apis/noobaa/v1alpha1/noobaa_types.go index 829621dda..648b33536 100644 --- a/pkg/apis/noobaa/v1alpha1/noobaa_types.go +++ b/pkg/apis/noobaa/v1alpha1/noobaa_types.go @@ -385,6 +385,10 @@ type NooBaaStatus struct { // BeforeUpgradeDbImage is the db image used before last db upgrade // +optional BeforeUpgradeDbImage *string `json:"beforeUpgradeDbImage,omitempty"` + + // PostgresMajorVersion is the major version of postgress db image + // +optional + PostgresMajorVersion PostgresMajorVersion `json:"postgresMajorVersion,omitempty"` } // SystemPhase is a string enum type for system phases @@ -514,6 +518,12 @@ type EndpointsStatus struct { VirtualHosts []string `json:"virtualHosts"` } +type PostgresMajorVersion string + +const ( + PostgresMajorVersionV16 PostgresMajorVersion = "16" +) + // UpgradePhase is a string enum type for upgrade phases type UpgradePhase string @@ -525,7 +535,7 @@ const ( UpgradePhaseMigrate UpgradePhase = "Migrating" - UpgradePhaseClean UpgradePhase = "Cleanning" + UpgradePhaseClean UpgradePhase = "Cleaning" UpgradePhaseFinished UpgradePhase = "DoneUpgrade" diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go index 5467257d9..145fcd85e 100644 --- a/pkg/bundle/deploy.go +++ b/pkg/bundle/deploy.go @@ -1415,7 +1415,7 @@ spec: status: {} ` -const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "e862d263d097ed43f774784eaaf9a616967746b67608fadbe4ca71d93b220ab6" +const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "5bfeed22b7201dfc6d56cda47e573dc0f5cac7350806b142c76509ad73b070b0" const File_deploy_crds_noobaa_io_noobaas_yaml = `--- apiVersion: apiextensions.k8s.io/v1 @@ -3330,6 +3330,10 @@ spec: description: Phase is a simple, high-level summary of where the System is in its lifecycle type: string + postgresMajorVersion: + description: PostgresMajorVersion is the major version of postgress + db image + type: string postgresUpdatePhase: description: Upgrade reports the status of the ongoing postgres upgrade process diff --git a/pkg/options/options.go b/pkg/options/options.go index ebb9e46b1..d524e88a0 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -4,6 +4,7 @@ import ( "github.com/noobaa/noobaa-operator/v5/pkg/util" "github.com/noobaa/noobaa-operator/v5/version" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -72,7 +73,11 @@ var NooBaaImage = ContainerImage // DBImage is the default db image url // it can be overridden for testing or different registry locations. -var DBImage = "quay.io/sclorg/postgresql-15-c9s" +var DBImage = "quay.io/sclorg/postgresql-16-c9s" + +// Postgress Major version is the major version of Postgress image +// This is referred for postgres upgrade. +var PostgresMajorVersion = nbv1.PostgresMajorVersionV16 // Psql12Image is the default postgres12 db image url // currently it can not be overridden. diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index 4f1e0d3f1..aa6183dad 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -123,6 +123,10 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { // create the db only if postgres secret is not given if r.NooBaa.Spec.ExternalPgSecret == nil { + if err := r.UpgradeDbPostgres16(); err != nil { + return err + } + if err := r.ReconcileDB(); err != nil { return err } @@ -131,6 +135,7 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { return err } } + // create bucket logging pvc if not provided by user for 'Guaranteed' logging in ODF env if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed { if err := r.ReconcileODFPersistentLoggingPVC( @@ -145,13 +150,13 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { // create notification log pvc if bucket notifications is enabled and pvc was not set explicitly if r.NooBaa.Spec.BucketNotifications.Enabled { - if err := r.ReconcileODFPersistentLoggingPVC( - "bucketNotifications.pvc", - "InvalidBucketNotificationConfiguration", - "Bucket notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'bucketNotifications.pvc'.", - r.NooBaa.Spec.BucketNotifications.PVC, - r.BucketNotificationsPVC); err != nil { - return err + if err := r.ReconcileODFPersistentLoggingPVC( + "bucketNotifications.pvc", + "InvalidBucketNotificationConfiguration", + "Bucket notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'bucketNotifications.pvc'.", + r.NooBaa.Spec.BucketNotifications.PVC, + r.BucketNotificationsPVC); err != nil { + return err } } @@ -171,6 +176,128 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { return nil } +func (r *Reconciler) UpgradeDbPostgres16() error { + var err error = nil + + if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseFinished { + r.Logger.Infof("UpgradePostgresDB: DB is already upgraded to postgresql-16") + return nil + } + + switch r.NooBaa.Status.PostgresUpdatePhase { + case "": + // Check whether it is fresh install case + sts := util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet) + sts.Name = "noobaa-db-pg" + sts.Namespace = r.NooBaa.Namespace + stsObj, err := util.KubeGetObject(sts) + // Postgres database doesn't exists, no need to upgrade. + if err != nil { + r.Logger.Infof("UpgradePostgresDB: old STS doesn't exist - no need for upgrade") + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + } else { + desiredImage := GetDesiredDBImage(r.NooBaa, "") + // Check whether upgrade is reqd, set phase depending the requirement + if r.isPostgresUpgradeReqd(desiredImage, stsObj.(*appsv1.StatefulSet)) { + r.Logger.Infof("UpgradePostgresDB: upgrading postgres16") + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhasePrepare + } else { + if r.NooBaa.Status.PostgresMajorVersion == nbv1.PostgresMajorVersionV16 { + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } else { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone + } + } + } + case nbv1.UpgradePhaseNone: + // Check whether it is fresh install case + sts := util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet) + sts.Name = "noobaa-db-pg" + sts.Namespace = r.NooBaa.Namespace + obj, err := util.KubeGetObject(sts) + if err != nil { + r.Logger.Infof("UpgradePostgresDB: STS doesn't exist - no need for upgrade") + return err + } + desiredImage := GetDesiredDBImage(r.NooBaa, "") + // Check wheter upgrade is reqd, set phase depending the requirement + if r.isPostgresUpgradeReqd(desiredImage, obj.(*appsv1.StatefulSet)) { + r.Logger.Infof("UpgradePostgresDB: upgrading postgres16") + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhasePrepare + } else { + if r.NooBaa.Status.PostgresMajorVersion == nbv1.PostgresMajorVersionV16 { + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } else { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone + } + } + case nbv1.UpgradePhasePrepare: + if err := r.setEndpointsDeploymentReplicas(0); err != nil { + r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err) + return err + } + desiredImage := GetDesiredDBImage(r.NooBaa, "") + if err = r.ReconcileObject( + r.NooBaaPostgresDB, func() error { + r.updateDBImageForUpgrade(desiredImage) + r.setPGUpgradeEnvInSTS() + return nil + }, + ); err != nil { + r.Logger.Errorf("got error on postgres STS reconcile %v", err) + break + } + + // Update the DB pod + restartError := r.RestartDbPods() + if restartError != nil { + r.Logger.Warn("UpgradePostgresDB: Unable to restart db pods") + } + + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseUpgrade + case nbv1.UpgradePhaseUpgrade: + dbPod := &corev1.Pod{} + dbPod.Name = "noobaa-db-pg-0" + dbPod.Namespace = r.NooBaaPostgresDB.Namespace + if !util.KubeCheckQuiet(dbPod) { + return nil + } + // make sure previous step has finished + if dbPod.ObjectMeta.DeletionTimestamp != nil { + r.Logger.Infof("UpgradePostgresDB: upgrade-db is not yet running, phase is: %s and deletion time stamp is %v", + dbPod.Status.Phase, dbPod.ObjectMeta.DeletionTimestamp) + return nil + } + r.NooBaa.Status.PostgresMajorVersion = options.PostgresMajorVersion + // Set the upgrade finished flag here and refer the same to remove the + // POSTGRES_UPGRADE flag in next reconcile. + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseClean + case nbv1.UpgradePhaseClean: + if err = r.ReconcileObject(r.NooBaaPostgresDB, func() error { + r.unSetPGUpgradeEnvInSTS() + return nil + }); err != nil { + r.Logger.Infof("UpgradePostgresDB: Cleaning of POSTGRES_UPGRADE env failed: err %v, retry", err) + return nil + } + + if err := r.setEndpointsDeploymentReplicas(1); err != nil { + r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err) + return err + } + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } + + if err := r.UpdateStatus(); err != nil { + return err + } + return err +} + // SetDesiredServiceAccount updates the ServiceAccount as desired for reconciling func (r *Reconciler) SetDesiredServiceAccount() error { if r.ServiceAccount.Annotations == nil { @@ -242,6 +369,94 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error { return nil } +func (r *Reconciler) updateDBImageForUpgrade(dbImage string) { + var podSpec = &r.NooBaaPostgresDB.Spec.Template.Spec + + for i := range podSpec.Containers { + c := &podSpec.Containers[i] + if c.Name == "db" { + // Fetch the DB image from options.go + c.Image = dbImage + if r.NooBaa.Spec.DBResources != nil { + c.Resources = *r.NooBaa.Spec.DBResources + } + + c.Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "pg_ctl -D /var/lib/pgsql/data/userdata/ -w -t 60 -m fast stop"}, + }, + }, + } + + } + } +} + +// SetEndpointsDeploymentReplicas updates the number of replicas on the endpoints deployment +func (r *Reconciler) setEndpointsDeploymentReplicas(replicas int32) error { + r.Logger.Infof("UpgradeMigrateDB:: setting endpoints replica count to %d", replicas) + return r.ReconcileObject(r.DeploymentEndpoint, func() error { + r.DeploymentEndpoint.Spec.Replicas = &replicas + return nil + }) +} + +// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade +func (r *Reconciler) isPostgresUpgradeReqd(desiredImage string, sts *appsv1.StatefulSet) bool { + var currentImage string + + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == "db" { + currentImage = container.Image + } + } + if currentImage != desiredImage && + r.NooBaa.Status.PostgresMajorVersion != nbv1.PostgresMajorVersionV16 { + return true + } + return false +} + +// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade +func (r *Reconciler) setPGUpgradeEnvInSTS() { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + for _, env := range container.Env { + if env.Name == "POSTGRESQL_UPGRADE" { + return + } + } + envVars := container.Env + newEnvVar := corev1.EnvVar{ + Name: "POSTGRESQL_UPGRADE", + Value: "copy", + } + + envVars = append(envVars, newEnvVar) + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars + return + } + } +} + +// unSet env "POSTGRESQL_UPGRADE=copy" in PG sts after upgrade +func (r *Reconciler) unSetPGUpgradeEnvInSTS() { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + newEnvVars := []corev1.EnvVar{} + for _, env := range container.Env { + if env.Name != "POSTGRESQL_UPGRADE" { + newEnvVars = append(newEnvVars, env) + } + } + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars + return + } + } + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone +} + // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling func (r *Reconciler) SetDesiredNooBaaDB() error { var NooBaaDBTemplate *appsv1.StatefulSet = nil @@ -472,10 +687,10 @@ func (r *Reconciler) setDesiredCoreEnv(c *corev1.Container) { if r.NooBaa.Spec.BucketNotifications.Enabled { envVar := corev1.EnvVar{ - Name: "NOTIFICATION_LOG_DIR", + Name: "NOTIFICATION_LOG_DIR", Value: "/var/logs/notifications", } - util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}); + util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}) } } @@ -555,10 +770,10 @@ func (r *Reconciler) SetDesiredCoreApp() error { }} util.MergeVolumeMountList(&c.VolumeMounts, ¬ificationVolumeMounts) - notificationVolumes := []corev1.Volume {{ + notificationVolumes := []corev1.Volume{{ Name: notificationsVolume, - VolumeSource: corev1.VolumeSource { - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource { + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: r.BucketNotificationsPVC.Name, }, }, @@ -1187,6 +1402,12 @@ func (r *Reconciler) reconcileDBRBAC() error { // ReconcileDB choose between different types of DB func (r *Reconciler) ReconcileDB() error { + if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhasePrepare || + r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseUpgrade || + r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseClean { + r.Logger.Infof("UpgradePostgresDB: Postgres Update in progress") + return nil + } if r.NooBaa.Spec.ExternalPgSecret != nil { return nil @@ -1217,6 +1438,7 @@ func (r *Reconciler) ReconcileDB() error { } } + return nil } @@ -1230,8 +1452,8 @@ func (r *Reconciler) ReconcileDBConfigMap(cm *corev1.ConfigMap, desiredFunc func return r.isObjectUpdated(result), nil } -//ReconcileODFPersistentLoggingPVC ensures a persistent logging pvc (either for bucket logging or bucket notificatoins) -//is properly configured. If needed and possible, allocate one from CephFS +// ReconcileODFPersistentLoggingPVC ensures a persistent logging pvc (either for bucket logging or bucket notificatoins) +// is properly configured. If needed and possible, allocate one from CephFS func (r *Reconciler) ReconcileODFPersistentLoggingPVC( fieldName string, errorName string, @@ -1243,21 +1465,21 @@ func (r *Reconciler) ReconcileODFPersistentLoggingPVC( // Return if persistent logging PVC already exists if pvc != nil { - pvc.Name = *pvcName; + pvc.Name = *pvcName log.Infof("PersistentLoggingPVC %s already exists and supports RWX access mode. Skipping ReconcileODFPersistentLoggingPVC.", *pvcName) return nil } util.KubeCheck(pvc) if pvc.UID != "" { - log.Infof("Persistent logging PVC %s already exists. Skipping creation.", *pvcName ) + log.Infof("Persistent logging PVC %s already exists. Skipping creation.", *pvcName) return nil } if !r.preparePersistentLoggingPVC(pvc, fieldName) { return util.NewPersistentError(errorName, errorText) } - r.Own(pvc); + r.Own(pvc) log.Infof("Persistent logging PVC %s does not exist. Creating...", *pvcName) err := r.Client.Create(r.Ctx, pvc) @@ -1269,7 +1491,7 @@ func (r *Reconciler) ReconcileODFPersistentLoggingPVC( } -//prepare persistent logging pvc +// prepare persistent logging pvc func (r *Reconciler) preparePersistentLoggingPVC(pvc *corev1.PersistentVolumeClaim, fieldName string) bool { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} @@ -1282,9 +1504,9 @@ func (r *Reconciler) preparePersistentLoggingPVC(pvc *corev1.PersistentVolumeCla if util.KubeCheck(sc) { r.Logger.Infof("%s not provided, defaulting to 'cephfs' storage class %s to create persistent logging pvc", fieldName, sc.Name) pvc.Spec.StorageClassName = &sc.Name - return true; + return true } else { - return false; + return false } } diff --git a/pkg/util/util.go b/pkg/util/util.go index 3011e2faf..a5b5b72a7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -623,6 +623,15 @@ func KubeGet(obj client.Object) (name string, kind string, err error) { return name, kind, err } +// KubeGetObject gets a client.Object, fills the given object and returns +func KubeGetObject(obj client.Object) (client.Object, error) { + klient := KubeClient() + objKey := ObjectKey(obj) + err := klient.Get(ctx, objKey, obj) + + return obj, err +} + // KubeList returns a list of objects. func KubeList(list client.ObjectList, options ...client.ListOption) bool { klient := KubeClient()