Add retry logic when an IMEX domain's channels can't be advertized

This indicates a temporariy failure that will be resolved if fewer
clique ids are used in the future for the same domain.

Signed-off-by: Kevin Klues <[email protected]>

cmd/nvidia-dra-controller/imex.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"sync"
@@ -41,8 +42,13 @@ const (
 	ImexDomainLabel               = "nvidia.com/gpu.imex-domain"
 	ResourceSliceImexChannelLimit = 128
 	DriverImexChannelLimit        = 2048
+	RetryTimeout                  = 1 * time.Minute
 )
 
+// RetryableError defines an error indicating that the operation returning the
+// error should be retried
+type retryableError struct{ error }
+
 // imexDomainOffsets represents the offset for assigning IMEX channels
 // to ResourceSlices for each <imex-domain, cliqueid> combination.
 type imexDomainOffsets map[string]map[string]int
@@ -51,6 +57,7 @@ type ImexManager struct {
 	driverName                    string
 	resourceSliceImexChannelLimit int
 	driverImexChannelLimit        int
+	retryTimeout                  time.Duration
 	waitGroup                     sync.WaitGroup
 	clientset                     kubernetes.Interface
 	imexDomainOffsets             imexDomainOffsets
@@ -95,6 +102,7 @@ func StartIMEXManager(ctx context.Context, config *Config) (*ImexManager, error)
 		driverName:                    DriverName,
 		resourceSliceImexChannelLimit: ResourceSliceImexChannelLimit,
 		driverImexChannelLimit:        DriverImexChannelLimit,
+		retryTimeout:                  RetryTimeout,
 		clientset:                     clientset,
 		owner:                         owner,
 		driverResources:               driverResources,
@@ -133,14 +141,26 @@ func (m *ImexManager) manageResourceSlices(ctx context.Context) error {
 				klog.Infof("Adding channels for new IMEX domain: %v", addedDomain)
 				if err := m.addImexDomain(addedDomain); err != nil {
 					klog.Errorf("Error adding channels for IMEX domain %s: %v", addedDomain, err)
-					return
+					if errors.As(err, &retryableError{}) {
+						klog.Infof("Retrying adding channels for IMEX domain %s after %v", addedDomain, m.retryTimeout)
+						go func() {
+							time.Sleep(m.retryTimeout)
+							addedDomainsCh <- addedDomain
+						}()
+					}
 				}
 				controller.Update(m.driverResources)
 			case removedDomain := <-removedDomainsCh:
 				klog.Infof("Removing channels for removed IMEX domain: %v", removedDomain)
 				if err := m.removeImexDomain(removedDomain); err != nil {
 					klog.Errorf("Error removing channels for IMEX domain %s: %v", removedDomain, err)
-					return
+					if errors.As(err, &retryableError{}) {
+						klog.Infof("Retrying removing channels for IMEX domain %s after %v", removedDomain, m.retryTimeout)
+						go func() {
+							time.Sleep(m.retryTimeout)
+							removedDomainsCh <- removedDomain
+						}()
+					}
 				}
 				controller.Update(m.driverResources)
 			case <-ctx.Done():
@@ -175,7 +195,7 @@ func (m *ImexManager) addImexDomain(imexDomain string) error {
 	}
 	offset, err := m.imexDomainOffsets.add(imexDomainID, cliqueID, m.resourceSliceImexChannelLimit, m.driverImexChannelLimit)
 	if err != nil {
-		return fmt.Errorf("error setting offset for IMEX channels: %w", err)
+		return retryableError{fmt.Errorf("error setting offset for IMEX channels: %w", err)}
 	}
 	m.driverResources = m.driverResources.DeepCopy()
 	m.driverResources.Pools[imexDomain] = generateImexChannelPool(imexDomain, offset, m.resourceSliceImexChannelLimit)