diff --git a/Makefile b/Makefile index 7f0d6fc2940..9ec58a01bcf 100644 --- a/Makefile +++ b/Makefile @@ -287,10 +287,9 @@ run-hashicorp-e2e-tests: GINKGO_FLAGS += --label-filter="end-to-end && !performa run-hashicorp-e2e-tests: test .PHONY: run-kube-e2e-tests -run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the Kubernetes E2E Tests in the {KUBE2E_TESTS} package +run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the legacy Kubernetes E2E Tests in the {KUBE2E_TESTS} package run-kube-e2e-tests: test - #---------------------------------------------------------------------------------- # Go Tests #---------------------------------------------------------------------------------- @@ -1096,6 +1095,9 @@ endif # distroless images CLUSTER_NAME ?= kind INSTALL_NAMESPACE ?= gloo-system +kind-setup: + VERSION=${VERSION} CLUSTER_NAME=${CLUSTER_NAME} ./ci/kind/setup-kind.sh + kind-load-%-distroless: kind load docker-image $(IMAGE_REGISTRY)/$*:$(VERSION)-distroless --name $(CLUSTER_NAME) diff --git a/changelog/v1.18.0-beta35/validate-large-configs.yaml b/changelog/v1.18.0-beta35/validate-large-configs.yaml new file mode 100644 index 00000000000..fe46cdd4420 --- /dev/null +++ b/changelog/v1.18.0-beta35/validate-large-configs.yaml @@ -0,0 +1,9 @@ +changelog: + - type: FIX + issueLink: https://github.com/solo-io/solo-projects/issues/7089 + resolvesIssue: false + description: >- + Fix the validation of large configurations when using envoy validation. + Previously if the configuration grew too large translation would be blocked unless Transformation Validation was disabled. + This change passes the configuration as a file instead, using STDIN (/dev/fd/0) + avoiding the need to store the file on disk. \ No newline at end of file diff --git a/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md b/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md index 43fb88d4511..a488cc3ffca 100644 --- a/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md +++ b/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md @@ -948,7 +948,7 @@ options for configuring admission control / validation | `validationServerGrpcMaxSizeBytes` | [.google.protobuf.Int32Value](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/int-32-value) | By default, gRPC validation messages between gateway and gloo pods have a max message size of 100 MB. Setting this value sets the gRPC max message size in bytes for the gloo validation server. This should only be changed if necessary. If not included, the gRPC max message size will be the default of 100 MB. | | `serverEnabled` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as `warn_route_short_circuiting` and `disable_transformation_validation`. If not included, the validation server will be enabled. | | `warnMissingTlsSecret` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Allows configuring validation to report a missing TLS secret referenced by a SslConfig or UpstreamSslConfig as a warning instead of an error. This will allow for eventually consistent workloads, but will also permit the accidental deletion of secrets being referenced, which would cause disruption in traffic. | -| `fullEnvoyValidation` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Configures the Gloo translation loop to send the final product of translation through Envoy validation mode. This has an negative impact on the total translation throughput, but it helps ensure the configuration will not be nacked when served to Envoy. This feature is disabled by default and is not recommended for production deployments unless the performance implications are well understood and acceptable. | +| `fullEnvoyValidation` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Configures the Gloo translation loop to send the final product of translation through Envoy validation mode. This has an negative impact on the total translation throughput, but it helps ensure the configuration will not be nacked when served to Envoy. This feature is disabled by default and is not recommended for production deployments unless the performance implications are well understood and acceptable. Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. When enabling this feature, consider increasing the timeout for the validating webhook (`.Values.gateway.validation.webhook.timeoutSeconds`). | diff --git a/projects/envoyinit/pkg/runner/run.go b/projects/envoyinit/pkg/runner/run.go index 1fe96f24383..7ea336c528b 100644 --- a/projects/envoyinit/pkg/runner/run.go +++ b/projects/envoyinit/pkg/runner/run.go @@ -6,6 +6,7 @@ import ( "log" "os" "syscall" + "time" "github.com/rotisserie/eris" "github.com/solo-io/gloo/pkg/utils/cmdutils" @@ -30,8 +31,17 @@ const ( func RunEnvoyValidate(ctx context.Context, envoyExecutable, bootstrapConfig string) error { logger := contextutils.LoggerFrom(ctx) - validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-yaml", bootstrapConfig, "-l", "critical", "--log-format", "%v") - if err := validateCmd.Run(); err != nil { + logger.Debugf("starting full envoy validation with size %d", len(bootstrapConfig)) + + validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-path", "/dev/fd/0", + "-l", "critical", "--log-format", "%v") + validateCmd = validateCmd.WithStdin(bytes.NewBufferString(bootstrapConfig)) + + start := time.Now() + err := validateCmd.Run() + logger.Debugf("full envoy validation of %d size completed in %s", len(bootstrapConfig), time.Since(start)) + + if err != nil { if os.IsNotExist(err) { // log a warning and return nil; will allow users to continue to run Gloo locally without // relying on the Gloo container with Envoy already published to the expected directory diff --git a/projects/gloo/api/v1/settings.proto b/projects/gloo/api/v1/settings.proto index 945b35cebe3..519bb7217f9 100644 --- a/projects/gloo/api/v1/settings.proto +++ b/projects/gloo/api/v1/settings.proto @@ -920,6 +920,10 @@ message GatewayOptions { // // This feature is disabled by default and is not recommended for production deployments unless // the performance implications are well understood and acceptable. + // + // Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. + // When enabling this feature, consider increasing the timeout for the validating webhook + // (`.Values.gateway.validation.webhook.timeoutSeconds`). google.protobuf.BoolValue full_envoy_validation = 14; } diff --git a/projects/gloo/pkg/api/v1/settings.pb.go b/projects/gloo/pkg/api/v1/settings.pb.go index b6e64cfadad..9b787e80b59 100644 --- a/projects/gloo/pkg/api/v1/settings.pb.go +++ b/projects/gloo/pkg/api/v1/settings.pb.go @@ -3445,6 +3445,10 @@ type GatewayOptions_ValidationOptions struct { // // This feature is disabled by default and is not recommended for production deployments unless // the performance implications are well understood and acceptable. + // + // Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. + // When enabling this feature, consider increasing the timeout for the validating webhook + // (`.Values.gateway.validation.webhook.timeoutSeconds`). FullEnvoyValidation *wrapperspb.BoolValue `protobuf:"bytes,14,opt,name=full_envoy_validation,json=fullEnvoyValidation,proto3" json:"full_envoy_validation,omitempty"` } diff --git a/test/kube2e/README.md b/test/kube2e/README.md index bacb04c8b16..65e4dbf13d2 100644 --- a/test/kube2e/README.md +++ b/test/kube2e/README.md @@ -2,6 +2,10 @@ > This directory houses legacy tests. All new tests should instead be added to the `test/kubernetes/e2e` directory. # Kubernetes End-to-End tests + +> These are our legacy Kubernetes E2E tests. We are migrating them to `../kubernetes/e2e`. Create new E2E tests there +> using the new framework. + See the [developer kube-e2e testing guide](/devel/testing/kube-e2e-tests.md) for more information about the philosophy of these tests. *Note: All commands should be run from the root directory of the Gloo repository* @@ -68,7 +72,7 @@ To run the regression tests, your kubeconfig file must point to a running Kubern Use the same command that CI relies on: ```bash -KUBE2E_TESTS= make run-kube-e2e-tests +CLUSTER_NAME=solo-test-cluster KUBE2E_TESTS= make run-kube-e2e-tests ``` #### Test Environment Variables @@ -81,6 +85,7 @@ The below table contains the environment variables that can be used to configure | WAIT_ON_FAIL | 0 | Set to 1 to prevent Ginkgo from cleaning up the Gloo Edge installation in case of failure. Useful to exec into inspect resources created by the test. A command to resume the test run (and thus clean up resources) will be logged to the output. | | TEAR_DOWN | false | Set to true to uninstall Gloo after the test suite completes | | RELEASED_VERSION | '' | Used by nightlies to tests a specific released version. 'LATEST' will find the latest release | +| CLUSTER_NAME | kind | Used to control which Kind cluster to run the tests inside | #### Common Test Errors `getting Helm chart version: expected a single entry with name [gloo], found: 5`\ diff --git a/test/kube2e/gateway/gateway_suite_test.go b/test/kube2e/gateway/gateway_suite_test.go index f98a9db07b3..60998adee87 100644 --- a/test/kube2e/gateway/gateway_suite_test.go +++ b/test/kube2e/gateway/gateway_suite_test.go @@ -22,6 +22,7 @@ import ( "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kube2e" "github.com/solo-io/gloo/test/kube2e/helper" + testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" skhelpers "github.com/solo-io/solo-kit/test/helpers" . "github.com/onsi/ginkgo/v2" @@ -73,7 +74,8 @@ func StartTestHelper() { } // We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup - clusterContext := cluster.MustKindContext("kind") + runtimeContext := testruntime.NewContext() + clusterContext := cluster.MustKindContext(runtimeContext.ClusterName) resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig) Expect(err).NotTo(HaveOccurred(), "can create kube resource client set") diff --git a/test/kube2e/gloo/gloo_suite_test.go b/test/kube2e/gloo/gloo_suite_test.go index e8f13f8bc7f..f269f3ddf1c 100644 --- a/test/kube2e/gloo/gloo_suite_test.go +++ b/test/kube2e/gloo/gloo_suite_test.go @@ -23,6 +23,7 @@ import ( "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kube2e" "github.com/solo-io/gloo/test/kube2e/helper" + testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" glootestutils "github.com/solo-io/gloo/test/testutils" "github.com/solo-io/go-utils/testutils" @@ -73,7 +74,8 @@ var _ = BeforeSuite(func() { } // We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup - clusterContext := cluster.MustKindContext("kind") + runtimeContext := testruntime.NewContext() + clusterContext := cluster.MustKindContext(runtimeContext.ClusterName) resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig) Expect(err).NotTo(HaveOccurred(), "can create kube resource client set") diff --git a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go index 6fa18a103f3..ec8633ad006 100644 --- a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go +++ b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go @@ -72,3 +72,16 @@ func (s *testingSuite) TestRejectInvalidTransformation() { s.Assert().Contains(output, "Failed to parse response template: Failed to parse "+ "header template ':status': [inja.exception.parser_error] (at 1:92) expected statement close, got '%'") } + +// TestLargeConfiguration checks webhook accepts large configuration when fullEnvoyValidation=true +func (s *testingSuite) TestLargeConfiguration() { + s.T().Cleanup(func() { + err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.LargeConfiguration, "-n", + s.testInstallation.Metadata.InstallNamespace) + s.Assertions.NoError(err, "can delete large configuration") + }) + + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.LargeConfiguration, "-n", + s.testInstallation.Metadata.InstallNamespace) + s.Assert().NoError(err) +} diff --git a/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml b/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml new file mode 100644 index 00000000000..4a62c86c730 --- /dev/null +++ b/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml @@ -0,0 +1,1006 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin + namespace: full-envoy-validation-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin + namespace: full-envoy-validation-test +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + version: v1 + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin + containers: + - name: httpbin + image: docker.io/mccutchen/go-httpbin:v2.6.0 + imagePullPolicy: IfNotPresent + command: [ go-httpbin ] + args: + - "-port" + - "8080" + - "-max-duration" + - "600s" # override default 10s + ports: + - containerPort: 8080 + # Include curl container for e2e testing, allows sending traffic mediated by the proxy sidecar + - name: curl + image: curlimages/curl:7.83.1 + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" + imagePullPolicy: IfNotPresent + command: + - "tail" + - "-f" + - "/dev/null" + - name: hey + image: gcr.io/solo-public/docs/hey:0.1.4 + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin + namespace: full-envoy-validation-test + labels: + app: httpbin + service: httpbin +spec: + ports: + - name: http + port: 8000 + targetPort: 8080 + selector: + app: httpbin +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: default-httpbin-8000 + namespace: full-envoy-validation-test +spec: + kube: + serviceName: httpbin + serviceNamespace: full-envoy-validation-test + servicePort: 8000 +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: jwt-validation-company + namespace: full-envoy-validation-test +spec: + options: +--- +apiVersion: gateway.solo.io/v1 +kind: RouteOption +metadata: + name: jwt-route-ip + namespace: full-envoy-validation-test +spec: + options: + autoHostRewrite: true + prefixRewrite: /get + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header("server") == "Google Frontend" %}{{ body() }}{% else %}{% if header(":status") == "401" %}{"error":"Invalid Token","errorCode":"INVALID_TOKEN","message":"Invalid Token","statusCode":403}{% else if header(":status") == "429" %}{"status":"fail","data":{"error":"QUOTA_EXCEEDED","path": "{{ request_header(":path") }}" }}{% else %}{{ body() }}{% endif %}{% endif %}' + headers: + :status: + text: '{% if header("server") == "Google Frontend" %}{{ header(":status") }}{% else %}{% if header(":status") == "401" %}403{% else if header(":status") == "429" %}450{% else %}{{ header(":status") }}{% endif %}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: cors-company + namespace: full-envoy-validation-test +spec: + options: + cors: + allowCredentials: true + allowHeaders: + - origin, + - x-requested-with, + - accept, + - content-type, + - authorization, + - x-something-api-key, + - x-something-device-type, + - x-something-platform, + - x-something-app-version, + - x-something-platform-version, + - x-something-default-language, + - x-something-country-code-override, + - x-something-user-token, + - x-something-install-id, + - x-something-profile-id, + - x-something-install-date, + - x-something-context-override, + - x-something-is-kid-profile, + - x-something-cosed, + - x-px-block-error, + - x-something-debug, + - x-something-token-key-id + allowMethods: + - GET + - PUT + - POST + - DELETE + - PATCH + allowOrigin: + - http://localhost:3000 + allowOriginRegex: + - https://*.company.com + exposeHeaders: + - '*' + maxAge: 3628800s +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: jwt-decode-company + namespace: full-envoy-validation-test +spec: + options: + stagedTransformations: + early: + requestTransforms: + - clearRouteCache: true + requestTransformation: + transformationTemplate: + advancedTemplates: true + extractors: + bearer: + header: authorization + regex: Bearer.(.*)\.(.*)\.(.*) + subgroup: 2 + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + body: + text: '{% if existsIn(context(), "video")%}{"video":{% set seriesMediaIdValue="default" + %}{% set nextEpisodeMediaIdValue="default" %}{% set page = at(context(), + "video") %}{ {% for key, value in page %} {% if at(loop, "is_first") + and key != "seriesMediaId" and key != "mediaId" and key != "nextEpisodeMediaId" + %}{% set first_key=key %}{% set first_value=value %}{% endif %}{% + if key == "mediaId" and substring(value, 0, 21) == "transmission:matchid:" + %}"mediaId":"video:mcp:unexpected-live-match",{% else %}{% if + key == "mediaId" %}"{{key}}":{% if not isNumber(value) %}"{% endif + %}{{value}}{% if not isNumber(value) %}"{% endif %},{% endif %}{% + endif %}{% if key == "seriesMediaId" %}{% set seriesMediaIdValue=value + %}{% endif %}{% if key == "nextEpisodeMediaId" %}{% set nextEpisodeMediaIdValue=value + %}{% endif %}{% if key != "seriesMediaId" and key != "nextEpisodeMediaId" + and key != "mediaId" %}"{{key}}":{% if not isNumber(value) %}"{% + endif %}{{value}}{% if not isNumber(value) %}"{% endif %},{% endif + %}{% endfor %}"{{first_key}}":{% if not isNumber(first_value) + %}"{% endif %}{{first_value}}{% if not isNumber(first_value) %}"{% + endif %}{% if seriesMediaIdValue != "default" and seriesMediaIdValue + != "" %},"seriesMediaId":{% if not isNumber(seriesMediaIdValue)%}"{% + endif %}{{seriesMediaIdValue}}{% if not isNumber(seriesMediaIdValue) + %}"{% endif %}{% endif %}{% if nextEpisodeMediaIdValue != "default" + and seriesMediaIdValue != "default" and seriesMediaIdValue != + "" %},"nextEpisodeMediaId":{% if not isNumber(nextEpisodeMediaIdValue)%}"{% + endif %}{{nextEpisodeMediaIdValue}}{% if not isNumber(nextEpisodeMediaIdValue) + %}"{% endif %}{% endif %} }} {% else %}{{ context() }}{% endif + %}' + extractors: + country: + header: x-something-country-code + regex: (AR|BO|CL|CO|CR|DO|EC|GT|HN|MX|NI|PA|PE|PR|PY|SV|US|UY|VE) + subgroup: 1 + profile: + header: profile-id + regex: .*?"id.{3}(\w*[-_\w*]*).* + subgroup: 1 + profile-extractor: + header: profile-extended + regex: .*?"id.{3}(\w*[-_\w*]*).* + subgroup: 1 + sub: + header: sub-claim + regex: ^(.*\|)?(.*)$ + subgroup: 2 + subscription-id: + header: x-something-subscription-info + regex: .*"subscriptionId".*?"([^"]+)".* + subgroup: 1 + subscription-id-user-token: + header: x-something-subscription-info-user-token + regex: .*"subscriptionId".*?"([^"]+)".* + subgroup: 1 + subscription-tier: + header: x-something-subscription-info-user-token + regex: .*"subscriptionTier.{3}(\w*).* + subgroup: 1 + headers: + x-something-country-blocked: + text: '{% if extraction("country") == "" %}country_not_allowed{% + endif %}' + x-something-install-id: + text: '{% if request_header("x-something-install-id") != "" %}{{ request_header("x-something-install-id") + }}{% else %}{% if header("x-something-install-id-claim") != "" %}{{ + header("x-something-install-id-claim") }}{% else %}asdfasdf{% + endif %}{% endif %}' + x-something-plan-group: + text: '{% if extraction("subscription-id") == "" %}default{% else + %}{%if extraction("subscription-id") in ["something-sv-web-prepaid-7d", + "something-gt-web-prepaid-7d", "something-gt-web-prepaid-15d", "something-gt-web-prepaid-30d", + "something-hd-web-prepaid-7d", "something-hd-web-prepaid-15d", "something-hd-web-prepaid-30d", + "something-ng-web-prepaid-7d", "something-ng-web-prepaid-15d","something-ng-web-prepaid-30d", + "something-pn-web-prepaid-7d", "something-hn-web-prepaid-standalone-7d"] + %}something-restricted{% else %}default{% endif %}{% endif%}' + x-something-plan-ids: + text: '{% if extraction("subscription-id") == "" %}{{ extraction("subscription-id-user-token") + }}{% else %}{{ extraction("subscription-id") }}{% endif %}' + x-something-profile-id: + text: '{% if request_header("x-something-profile-id") != "" %}{% set + a = false %}{% for num in range(length(header("x-something-available-profile-ids")) + / 36 ) %}{% if substring(header("x-something-available-profile-ids"), + num * 36 + num , 36) == request_header("x-something-profile-id") %}{{ + request_header("x-something-profile-id") }}{% set a = true %}{% endif + %}{% endfor %}{% if a == false %}{{ request_header("x-something-profile-id") + }}{% endif %}{% else %} {% if header("x-iss") == "identity-api.self.something.com"%} {{ + extraction("profile") }}{% else %}{% if extraction("profile-extractor") + != "" %}{{ extraction("profile-extractor") }}{% else %}{{ extraction("user-token-profile-id") + }}{% endif%}{% endif %}{% endif %}' + x-something-subscription-plan-tier: + text: '{{ extraction("subscription-tier") }}' + x-something-user-id: + text: '{{ extraction("sub") }}' + headersToRemove: + - authorization + - x-something-subscription-info + - x-something-install-id-claim + - profile-extended + - x-something-subscription-info-user-token +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-1 + namespace: full-envoy-validation-test +spec: + virtualHost: + domains: + - httpbin-1.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + namespace: full-envoy-validation-test + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + namespace: full-envoy-validation-test + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + namespace: full-envoy-validation-test +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-2 +spec: + virtualHost: + domains: + - httpbin-2.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-3 +spec: + virtualHost: + domains: + - httpbin-3.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: default-httpbin-8000 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/validation/types.go b/test/kubernetes/e2e/features/validation/types.go index 720c00ea9a0..f96cce41ea1 100644 --- a/test/kubernetes/e2e/features/validation/types.go +++ b/test/kubernetes/e2e/features/validation/types.go @@ -45,6 +45,9 @@ var ( VSTransformationHeaderText = filepath.Join(util.MustGetThisDir(), "testdata", "transformation", "vs-transform-header-text.yaml") VSTransformationSingleReplace = filepath.Join(util.MustGetThisDir(), "testdata", "transformation", "vs-transform-single-replace.yaml") + // Valid resources + LargeConfiguration = filepath.Join(util.MustGetThisDir(), "testdata", "valid-resources", "large-configuration.yaml") + // Split webhook validation BasicUpstream = filepath.Join(util.MustGetThisDir(), "testdata", "split-webhook", "basic-upstream.yaml") diff --git a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go index 510bcee937e..275d042bca8 100644 --- a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go @@ -221,13 +221,13 @@ func (s *testingSuite) TestRejectTransformation() { // this should be rejected output, err = s.testInstallation.Actions.Kubectl().ApplyFileWithOutput(s.ctx, validation.VSTransformationExtractors, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().Error(err) - s.Assert().Contains(output, "envoy validation mode output: error initializing configuration '': Failed to parse response template: group 1 requested for regex with only 0 sub groups") + s.Assert().Contains(output, "Failed to parse response template: group 1 requested for regex with only 0 sub groups") // Single replace mode -- rejects invalid subgroup in transformation // note that the regex has no subgroups, but we are trying to extract the first subgroup // this should be rejected output, err = s.testInstallation.Actions.Kubectl().ApplyFileWithOutput(s.ctx, validation.VSTransformationSingleReplace, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().Error(err) - s.Assert().Contains(output, "envoy validation mode output: error initializing configuration '': Failed to parse response template: group 1 requested for regex with only 0 sub groups") + s.Assert().Contains(output, "Failed to parse response template: group 1 requested for regex with only 0 sub groups") } diff --git a/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml b/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml index 2c13d97feb5..7a0afeef60c 100644 --- a/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml @@ -1,7 +1,9 @@ gateway: validation: failurePolicy: Fail # For "strict" validation mode, fail the validation if webhook server is not available - allowWarnings: false # For "strict" validation mode, webhook will also reject warnings + allowWarnings: true # These tests to not need to fail on warnings # transformation validation is disabled because full envoy validation is enabled. disableTransformationValidation: true + webhook: + timeoutSeconds: 30 # We are seeing Envoy take 10s of seconds to validate some of the larger configurations fullEnvoyValidation: true