Implement shoot migration for firewall-controller-manager.

pkg/controller/worker/actuator.go

@@ -2,6 +2,8 @@ package worker
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/gardener/gardener/extensions/pkg/util"
 
@@ -12,10 +14,14 @@ import (
 	apismetal "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal"
 	"github.com/metal-stack/gardener-extension-provider-metal/pkg/imagevector"
 	"github.com/metal-stack/gardener-extension-provider-metal/pkg/metal"
+	metalclient "github.com/metal-stack/gardener-extension-provider-metal/pkg/metal/client"
 	metalv1alpha1 "github.com/metal-stack/machine-controller-manager-provider-metal/pkg/provider/migration/legacy-api/machine/v1alpha1"
+	metalgo "github.com/metal-stack/metal-go"
+	"github.com/metal-stack/metal-go/api/models"
 
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	gardener "github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/metal-stack/metal-lib/pkg/cache"
 
 	"github.com/go-logr/logr"
 
@@ -28,35 +34,148 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-type delegateFactory struct {
-	logger logr.Logger
+type (
+	// actuator reconciles the cluster's worker nodes and the firewalls.
+	//
+	// why is the firewall reconciliation here and not in the controlplane controller?
+	// the controlplane controller deploys the firewall-controller-manager including validating and mutating webhooks
+	// this has to be running before we can create a firewall deployment because the mutating webhook is creating the userdata
+	// the worker controller acts after the controlplane controller, also the terms and responsibilities are pretty similar between machine-controller-manager and firewall-controller-manager,
+	// so this place seems to be a valid fit.
+	actuator struct {
+		workerActuator worker.Actuator
+
+		logger           logr.Logger
+		controllerConfig config.ControllerConfiguration
+		networkCache     *cache.Cache[*cacheKey, *models.V1NetworkResponse]
+
+		client  client.Client
+		scheme  *runtime.Scheme
+		decoder runtime.Decoder
+	}
+
+	delegateFactory struct {
+		logger logr.Logger
+
+		restConfig *rest.Config
+
+		client  client.Client
+		scheme  *runtime.Scheme
+		decoder runtime.Decoder
+
+		dataGetter func(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) (*additionalData, error)
+
+		machineImageMapping []config.MachineImage
+	}
+
+	workerDelegate struct {
+		logger logr.Logger
 
-	restConfig *rest.Config
+		client  client.Client
+		scheme  *runtime.Scheme
+		decoder runtime.Decoder
 
-	client  client.Client
-	scheme  *runtime.Scheme
-	decoder runtime.Decoder
+		machineImageMapping []config.MachineImage
+		seedChartApplier    gardener.ChartApplier
+		serverVersion       string
+
+		cluster *extensionscontroller.Cluster
+		worker  *extensionsv1alpha1.Worker
+
+		machineClasses     []map[string]interface{}
+		machineDeployments worker.MachineDeployments
+		machineImages      []apismetal.MachineImage
+
+		additionalData *additionalData
+	}
+)
+
+func (a *actuator) InjectScheme(scheme *runtime.Scheme) error {
+	a.scheme = scheme
+	a.decoder = serializer.NewCodecFactory(a.scheme).UniversalDecoder()
+	return nil
+}
 
-	machineImageMapping []config.MachineImage
-	controllerConfig    config.ControllerConfiguration
+func (a *actuator) InjectClient(client client.Client) error {
+	a.client = client
+	return nil
 }
 
 // NewActuator creates a new Actuator that updates the status of the handled WorkerPoolConfigs.
 func NewActuator(machineImages []config.MachineImage, controllerConfig config.ControllerConfiguration) worker.Actuator {
+	logger := log.Log.WithName("metal-worker-actuator")
+
+	a := &actuator{
+		logger:           logger,
+		controllerConfig: controllerConfig,
+		networkCache: cache.New(15*time.Minute, func(ctx context.Context, accessor *cacheKey) (*models.V1NetworkResponse, error) {
+			mclient, ok := ctx.Value("client").(metalgo.Client)
+			if !ok {
+				return nil, fmt.Errorf("no client passed in context")
+			}
+
+			privateNetwork, err := metalclient.GetPrivateNetworkFromNodeNetwork(ctx, mclient, accessor.projectID, accessor.nodeCIDR)
+			if err != nil {
+				return nil, err
+			}
+
+			return privateNetwork, nil
+		}),
+	}
+
 	delegateFactory := &delegateFactory{
 		logger:              log.Log.WithName("worker-actuator"),
 		machineImageMapping: machineImages,
-		controllerConfig:    controllerConfig,
+		dataGetter:          a.getAdditionalData,
 	}
-	return genericactuator.NewActuator(
-		log.Log.WithName("metal-worker-actuator"),
+
+	a.workerActuator = genericactuator.NewActuator(
+		logger,
 		delegateFactory,
 		metal.MachineControllerManagerName,
 		mcmChart,
 		mcmShootChart,
 		imagevector.ImageVector(),
 		extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot),
 	)
+
+	return a
+}
+
+func (a *actuator) Reconcile(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.firewallReconcile(ctx, worker, cluster)
+	if err != nil {
+		return err
+	}
+
+	return a.workerActuator.Reconcile(ctx, worker, cluster)
+}
+
+func (a *actuator) Delete(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.workerActuator.Delete(ctx, worker, cluster)
+	if err != nil {
+		return err
+	}
+
+	return a.firewallDelete(ctx, cluster)
+}
+
+func (a *actuator) Migrate(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.workerActuator.Migrate(ctx, worker, cluster)
+	if err != nil {
+		return err
+	}
+
+	return a.firewallMigrate(ctx, cluster)
+}
+
+func (a *actuator) Restore(ctx context.Context, worker *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster) error {
+	err := a.firewallRestore(ctx, worker, cluster)
+	if err != nil {
+		return err
+	}
+
+	return a.workerActuator.Restore(ctx, worker, cluster)
 }
 
 func (d *delegateFactory) InjectScheme(scheme *runtime.Scheme) error {
@@ -95,70 +214,24 @@ func (d *delegateFactory) WorkerDelegate(ctx context.Context, worker *extensions
 		return nil, err
 	}
 
-	return NewWorkerDelegate(
-		d.logger,
-		d.client,
-		d.scheme,
-		d.decoder,
-
-		d.machineImageMapping,
-		seedChartApplier,
-		serverVersion.GitVersion,
-
-		worker,
-		cluster,
-		d.controllerConfig,
-	), nil
-}
-
-type workerDelegate struct {
-	logger logr.Logger
-
-	client  client.Client
-	scheme  *runtime.Scheme
-	decoder runtime.Decoder
-
-	machineImageMapping []config.MachineImage
-	seedChartApplier    gardener.ChartApplier
-	serverVersion       string
-
-	cluster *extensionscontroller.Cluster
-	worker  *extensionsv1alpha1.Worker
-
-	machineClasses     []map[string]interface{}
-	machineDeployments worker.MachineDeployments
-	machineImages      []apismetal.MachineImage
-
-	controllerConfig config.ControllerConfiguration
-}
-
-// NewWorkerDelegate creates a new context for a worker reconciliation.
-func NewWorkerDelegate(
-	logger logr.Logger,
-	client client.Client,
-	scheme *runtime.Scheme,
-	decoder runtime.Decoder,
-
-	machineImageMapping []config.MachineImage,
-	seedChartApplier gardener.ChartApplier,
-	serverVersion string,
-
-	worker *extensionsv1alpha1.Worker,
-	cluster *extensionscontroller.Cluster,
-	controllerConfig config.ControllerConfiguration,
+	additionalData, err := d.dataGetter(ctx, worker, cluster)
+	if err != nil {
+		return nil, err
+	}
 
-) genericactuator.WorkerDelegate {
 	return &workerDelegate{
-		logger:  logger,
-		client:  client,
-		scheme:  scheme,
-		decoder: decoder,
+		logger:  d.logger,
+		client:  d.client,
+		scheme:  d.scheme,
+		decoder: d.decoder,
 
-		machineImageMapping: machineImageMapping,
+		machineImageMapping: d.machineImageMapping,
 		seedChartApplier:    seedChartApplier,
-		serverVersion:       serverVersion,
+		serverVersion:       serverVersion.GitVersion,
 
 		cluster: cluster,
 		worker:  worker,
-	}
+
+		additionalData: additionalData,
+	}, nil
 }

pkg/controller/worker/firewall_delete.go

@@ -0,0 +1,50 @@
+package worker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller"
+	retryutils "github.com/gardener/gardener/pkg/utils/retry"
+	fcmv2 "github.com/metal-stack/firewall-controller-manager/api/v2"
+	"github.com/metal-stack/gardener-extension-provider-metal/pkg/metal"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (a *actuator) firewallDelete(ctx context.Context, cluster *extensionscontroller.Cluster) error {
+	a.logger.Info("ensuring firewall deployment gets deleted")
+
+	return retryutils.UntilTimeout(ctx, 5*time.Second, 2*time.Minute, func(ctx context.Context) (bool, error) {
+		deploy := &fcmv2.FirewallDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      metal.FirewallDeploymentName,
+				Namespace: cluster.ObjectMeta.Name,
+			},
+		}
+
+		err := a.client.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				a.logger.Info("firewall deployment deletion succeeded")
+				return retryutils.Ok()
+			}
+			return retryutils.SevereError(fmt.Errorf("error getting firewall deployment: %w", err))
+		}
+
+		if deploy.DeletionTimestamp == nil {
+			a.logger.Info("deleting firewall deployment")
+			err = a.client.Delete(ctx, deploy)
+			if err != nil {
+				return retryutils.SevereError(fmt.Errorf("error deleting firewall deployment: %w", err))
+			}
+
+			return retryutils.MinorError(errors.New("firewall deployment is still ongoing"))
+		}
+
+		return retryutils.MinorError(errors.New("firewall deployment is still ongoing"))
+	})
+}