From 2b6de20ce156849f15b6e2c4ea5fd567e61567aa Mon Sep 17 00:00:00 2001 From: An Tran Date: Fri, 14 Jun 2024 11:43:09 +1000 Subject: [PATCH 1/2] Add support to set proxy buffer size --- CHANGELOG.md | 2 + doc/parameters.md | 7 +++ gateway/apicast.d/buffers.conf | 5 ++ t/proxy-buffers.t | 86 ++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 gateway/apicast.d/buffers.conf create mode 100644 t/proxy-buffers.t diff --git a/CHANGELOG.md b/CHANGELOG.md index 31def373e..22056f0e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Support Financial-grade API (FAPI) - Baseline profile [PR #1465](https://github.com/3scale/APIcast/pull/1465) [THREESCALE-10973](https://issues.redhat.com/browse/THREESCALE-10973) +- Added `APICAST_PROXY_BUFFER_SIZE` variable to allow configure the size of the buffer used for handling the response received from the proxied server. [PR #1473](https://github.com/3scale/APIcast/pull/1473), [THREESCALE-8410](https://issues.redhat.com/browse/THREESCALE-8410) + ## [3.15.0] 2024-04-04 ### Fixed diff --git a/doc/parameters.md b/doc/parameters.md index 362617b32..3acbbfe9d 100644 --- a/doc/parameters.md +++ b/doc/parameters.md @@ -524,6 +524,13 @@ directive](https://nginx.org/en/docs/http/ngx_http_core_module.html#large_client Sets the maximum size of shared memory used by batcher policy. The accepted [size units](https://github.com/openresty/lua-nginx-module?tab=readme-ov-file#lua_shared_dict) are k and m. +### `APICAST_PROXY_BUFFER_SIZE` + +**Default:** 4k|8k; +**Value:** string + +Sets the size of the buffer used for handling the response received from the proxied server. This variable will set both [`proxy_buffer` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) and [`proxy_buffer_size` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size). By default, the buffer size is equal to one memory page. This is either 4K or 8K, depending on a platform. + ### `OPENTELEMETRY` This environment variable enables NGINX instrumentation using OpenTelemetry tracing library. diff --git a/gateway/apicast.d/buffers.conf b/gateway/apicast.d/buffers.conf new file mode 100644 index 000000000..b100cbe02 --- /dev/null +++ b/gateway/apicast.d/buffers.conf @@ -0,0 +1,5 @@ +{%- assign proxy_buffer_size = env.APICAST_PROXY_BUFFER_SIZE %} +{% if proxy_buffer_size -%} + proxy_buffers 8 {{ proxy_buffer_size }}; + proxy_buffer_size {{ proxy_buffer_size }}; +{%- endif %} diff --git a/t/proxy-buffers.t b/t/proxy-buffers.t new file mode 100644 index 000000000..9643c1a1c --- /dev/null +++ b/t/proxy-buffers.t @@ -0,0 +1,86 @@ +use lib 't'; +use Test::APIcast::Blackbox 'no_plan'; + +run_tests(); + +__DATA__ + +=== TEST 1: reject with 502 when upstream return large header (the header exceed the size +of proxy_buffer_size) +--- configuration env +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "GET", "metric_system_name": "hits", "delta": 2 } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + location / { + content_by_lua_block { + ngx.header["X-Large-Header"] = string.rep("a", 2^12) + } + } +--- request +GET /?user_key=value +--- error_code: 502 +--- error_log eval +qr/upstream sent too big header while reading response header from upstream/ + + +=== TEST 2: large utream header with APICAST_PROXY_BUFFER_SIZE set to 8k +--- env eval +( + 'APICAST_PROXY_BUFFER_SIZE' => '8k', +) +--- configuration env +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "GET", "metric_system_name": "hits", "delta": 2 } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + location / { + content_by_lua_block { + ngx.header["X-Large-Header"] = string.rep("a", 2^12) + } + } +--- request +GET /?user_key=value +--- response_headers eval +"X-Large-Header: " . ("a" x 4096) . "\r\n\r\n" +--- error_code: 200 +--- no_error_log +[error] From 5b984c6341dd6719bb21e4440aacecb5e6be81ee Mon Sep 17 00:00:00 2001 From: An Tran Date: Fri, 12 Jul 2024 12:52:53 +1000 Subject: [PATCH 2/2] Adjust docs based on PR review feedback --- CHANGELOG.md | 2 +- doc/parameters.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22056f0e0..9d6dcb29e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Support Financial-grade API (FAPI) - Baseline profile [PR #1465](https://github.com/3scale/APIcast/pull/1465) [THREESCALE-10973](https://issues.redhat.com/browse/THREESCALE-10973) -- Added `APICAST_PROXY_BUFFER_SIZE` variable to allow configure the size of the buffer used for handling the response received from the proxied server. [PR #1473](https://github.com/3scale/APIcast/pull/1473), [THREESCALE-8410](https://issues.redhat.com/browse/THREESCALE-8410) +- Added the `APICAST_PROXY_BUFFER_SIZE` variable to allow configuration of the buffer size for handling response from the proxied servers. [PR #1473](https://github.com/3scale/APIcast/pull/1473), [THREESCALE-8410](https://issues.redhat.com/browse/THREESCALE-8410) ## [3.15.0] 2024-04-04 diff --git a/doc/parameters.md b/doc/parameters.md index 3acbbfe9d..083fa4df3 100644 --- a/doc/parameters.md +++ b/doc/parameters.md @@ -529,7 +529,7 @@ Sets the maximum size of shared memory used by batcher policy. The accepted [siz **Default:** 4k|8k; **Value:** string -Sets the size of the buffer used for handling the response received from the proxied server. This variable will set both [`proxy_buffer` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) and [`proxy_buffer_size` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size). By default, the buffer size is equal to one memory page. This is either 4K or 8K, depending on a platform. +Sets the size of the buffer used for handling the response received from the proxied server. This variable sets both [`proxy_buffer` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) and [`proxy_buffer_size` NGINX directive](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size). By default, the buffer size is equal to one memory page. This is either 4 KiB or 8 KiB, depending on a platform. ### `OPENTELEMETRY`