Collect Gloo metrics and some snapshots on test failure #10400
Conversation
| @@ -225,6 +225,8 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup | |||
| []string{"conflict with more specific or older VirtualHostOptions"}, | |||
| defaults.KubeGatewayReporter, | |||
| ) | |||
|
|
|||
| s.Assert().Equal(true, false, "intentionally failing to trigger drump, remove when done debugging") | |||
There was a problem hiding this comment.
I will be removing this after I have an artifact example to link.
| // Fetch the name of the Gloo Gateway controller pod | ||
| getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, | ||
| "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") | ||
| _ = getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() | ||
|
|
||
| // Clean up and check the output | ||
| glooPodName := strings.Trim(podStdOut.String(), "'") | ||
| if glooPodName == "" { | ||
| fmt.Printf("Failed to get the name of the Gloo Gateway controller pod: %s\n", podStdErr.String()) | ||
| return | ||
| } |
There was a problem hiding this comment.
Happy to make this it's own method if we think we will need it more than in this case.
|
Visit the preview URL for this PR (updated for commit e2b43ec): https://gloo-edge--pr10400-rolds-test-failure-c-kkld9hrb.web.app (expires Sat, 30 Nov 2024 01:54:49 GMT) 🔥 via Firebase Hosting GitHub Action 🌎 Sign: 77c2b86e287749579b7ff9cadb81e099042ef677 |
| // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller | ||
| metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, | ||
| "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", | ||
| "curl", "http://localhost:9091/metrics") | ||
| _ = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() | ||
| metricsFile.Close() | ||
|
|
||
| // Get krt snapshot from the Gloo Gateway controller pod and write it to a file | ||
| krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") | ||
| krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) | ||
| i.Assertions.Require.NoError(err) | ||
|
|
||
| // Using an ephemeral debug pod fetch the krt snapshot from the Gloo Gateway controller | ||
| krtSnapshotCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, | ||
| "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", | ||
| "curl", "http://localhost:9095/snapshots/krt") | ||
| _ = krtSnapshotCmd.WithStdout(krtSnapshotFile).WithStderr(krtSnapshotFile).Run() | ||
| krtSnapshotFile.Close() |
There was a problem hiding this comment.
while these make sense it would be ideal if we could add them to the main admin gloo endpoint tooling as its a pretty general case. I know its annoying to do it on a first pass (ie one case) but Im pretty sure I would love to have it on all cases
https://github.com/solo-io/gloo/blob/main/test/helpers/kube_dump.go#L46 is an example of our end dump and it includes hitting the stats endpoint like https://github.com/solo-io/gloo/blob/main/test/helpers/kube_dump.go#L311 which would be a nice setup that we can add to these types of tests
Description
I've encountered a couple of cases where this information would have been helpful in debugging
Interesting decisions
I wanted to use the ephemeral debug containers and had to first fetch the pod name because
kubectl debugdoesn't supportdeployment/glooas a pod.Testing steps
Take a look at the artifacts found this failed test.
Checklist: