Skip to content

Commit

Permalink
Fix operator panic when checking for pod crashloop status (#115)
Browse files Browse the repository at this point in the history
* check index out of bounds to prevent panic

* simplify ordered ready crash loop checks

* add else statement
  • Loading branch information
SatyaKuppam authored Oct 24, 2023
1 parent d6ae77c commit c20c2d4
Showing 1 changed file with 22 additions and 13 deletions.
35 changes: 22 additions & 13 deletions controllers/druid/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -657,22 +657,31 @@ func checkCrashStatus(ctx context.Context, sdk client.Client, drd *v1alpha1.Drui
return err
}

// the below condition evalutes if a pod is in
// 1. failed state 2. unknown state
// OR condtion.status is false which evalutes if neither of these conditions are met
// 1. ContainersReady 2. PodInitialized 3. PodReady 4. PodScheduled
for _, p := range podList {
if p.(*v1.Pod).Status.ContainerStatuses[0].RestartCount > 1 {
if p.(*v1.Pod).Status.Phase == v1.PodFailed || p.(*v1.Pod).Status.Phase == v1.PodUnknown {
err := writers.Delete(ctx, sdk, drd, p, emitEvents, &client.DeleteOptions{})
if err != nil {
return err
}
msg := fmt.Sprintf("Deleted pod [%s] in namespace [%s], since it was in [%s] state.", p.GetName(), p.GetNamespace(), p.(*v1.Pod).Status.Phase)
logger.Info(msg, "Object", stringifyForLogging(p, drd), "name", drd.Name, "namespace", drd.Namespace)
} else {
for _, condition := range p.(*v1.Pod).Status.Conditions {
// condition.type Ready means the pod is able to service requests
if condition.Type == v1.ContainersReady {
// the below condition evalutes if a pod is in
// 1. pending state 2. failed state 3. unknown state
// OR condtion.status is false which evalutes if neither of these conditions are met
// 1. ContainersReady 2. PodInitialized 3. PodReady 4. PodScheduled
if p.(*v1.Pod).Status.Phase != v1.PodRunning || condition.Status == v1.ConditionFalse {
err := writers.Delete(ctx, sdk, drd, p, emitEvents, &client.DeleteOptions{})
if err != nil {
return err
} else {
msg := fmt.Sprintf("Deleted pod [%s] in namespace [%s], since it was in crashloopback state.", p.GetName(), p.GetNamespace())
logger.Info(msg, "Object", stringifyForLogging(p, drd), "name", drd.Name, "namespace", drd.Namespace)
if condition.Status == v1.ConditionFalse {
for _, containerStatus := range p.(*v1.Pod).Status.ContainerStatuses {
if containerStatus.RestartCount > 1 {
err := writers.Delete(ctx, sdk, drd, p, emitEvents, &client.DeleteOptions{})
if err != nil {
return err
}
msg := fmt.Sprintf("Deleted pod [%s] in namespace [%s], since the container [%s] was crashlooping.", p.GetName(), p.GetNamespace(), containerStatus.Name)
logger.Info(msg, "Object", stringifyForLogging(p, drd), "name", drd.Name, "namespace", drd.Namespace)
}
}
}
}
Expand Down

0 comments on commit c20c2d4

Please sign in to comment.