[PATCH 00 of 14] request body discarding cleanup
Maxim Dounin
mdounin at mdounin.ru
Sat Apr 20 01:28:53 UTC 2024
Hello!
On Sat, Apr 20, 2024 at 03:59:48AM +0300, Maxim Dounin wrote:
> The following patch series is focused on various edge cases related
> to reading and discarding of a request body.
>
> Notably, it should be now safe (from the request body point of view)
> to configure complex processing for errors, such as 413 (Request Entity
> Too Large), including proxying, without additional precautions, such
> as "proxy_set_header Content-Length '';" and "proxy_pass_request_body off;".
>
> The series also fixes the issue observed with error_page 413 and HTTP/2
> (and HTTP/3), which previously required relaxed client_max_body_size
> limit in the error page location to work.
>
> Review and testing appreciated.
And here are tests:
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1712763700 -10800
# Wed Apr 10 18:41:40 2024 +0300
# Node ID 9e62f95165d75c2d78b07d88f46f8aeca57697b5
# Parent fe80aebae728b6e8fccafe1ded1e8ae12a0ac72d
Tests: fixed HTTP/3 stream offset adjustment when sending body.
Previously, size of the body without the DATA frame header was used,
resulting in incorrect offset being used in further QUIC frames, making it
impossible to send the request body in multiple DATA frames.
diff --git a/lib/Test/Nginx/HTTP3.pm b/lib/Test/Nginx/HTTP3.pm
--- a/lib/Test/Nginx/HTTP3.pm
+++ b/lib/Test/Nginx/HTTP3.pm
@@ -470,7 +470,7 @@ sub h3_body {
my $buf = pack_body($self, $body) if defined $body;
my $offset = $self->{streams}{$sid}{sent};
- $self->{streams}{$sid}{sent} += length($body);
+ $self->{streams}{$sid}{sent} += length($buf);
$self->raw_write($self->build_stream($buf,
start => $extra->{body_more}, sid => $sid, offset => $offset));
}
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1712763707 -10800
# Wed Apr 10 18:41:47 2024 +0300
# Node ID 5f664310b16b810e820880dd51d0151ce99a8464
# Parent 9e62f95165d75c2d78b07d88f46f8aeca57697b5
Tests: fixed HTTP/3 build_int() for large integers.
diff --git a/lib/Test/Nginx/HTTP3.pm b/lib/Test/Nginx/HTTP3.pm
--- a/lib/Test/Nginx/HTTP3.pm
+++ b/lib/Test/Nginx/HTTP3.pm
@@ -2087,14 +2087,14 @@ sub build_int {
} else {
pack("C*",
- build_int_set->($value, 7, 3),
- build_int_set->($value, 6, 0),
- build_int_set->($value, 5, 0),
- build_int_set->($value, 4, 0),
- build_int_set->($value, 3, 0),
- build_int_set->($value, 2, 0),
- build_int_set->($value, 1, 0),
- build_int_set->($value, 0, 0),
+ $build_int_set->($value, 7, 3),
+ $build_int_set->($value, 6, 0),
+ $build_int_set->($value, 5, 0),
+ $build_int_set->($value, 4, 0),
+ $build_int_set->($value, 3, 0),
+ $build_int_set->($value, 2, 0),
+ $build_int_set->($value, 1, 0),
+ $build_int_set->($value, 0, 0),
);
}
}
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1713557015 -10800
# Fri Apr 19 23:03:35 2024 +0300
# Node ID b6396847375982059a2a61992676ca0256279b03
# Parent 5f664310b16b810e820880dd51d0151ce99a8464
Tests: adjusted http_headers_multi.t for $content_length changes.
The $content_length variable is going to be not available after discarding
the request body. As such, the relevant location is now proxied, so the
request body is not discarded.
diff --git a/http_headers_multi.t b/http_headers_multi.t
--- a/http_headers_multi.t
+++ b/http_headers_multi.t
@@ -96,7 +96,7 @@ http {
add_header X-Host $host;
add_header X-Remote-User $remote_user;
- return 204;
+ proxy_pass http://127.0.0.1:8080/backend;
}
location /d {
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1713562558 -10800
# Sat Apr 20 00:35:58 2024 +0300
# Node ID 94d7d4030fccc84fa5d27230dc0e1dd2367e9194
# Parent b6396847375982059a2a61992676ca0256279b03
Tests: tests for usage of discarded body.
The client_max_body_size limit should be ignored when the request body
is already discarded. In HTTP/1.x, this is done by checking the
r->discard_body flag when the body is being discarded, and because
r->headers_in.content_length_n is 0 when it's already discarded. This,
however, does not happen with HTTP/2 and HTTP/3, and therefore
"error_page 413" does not work without relaxing the limit.
Further, with proxy_pass, r->headers_in.content_length_n is used to determine
length of the request body, and therefore is not correct if discarding of
the request body isn't yet complete. While discarding the request body,
r->headers_in.content_length_n contains the rest of the body to discard
(or, in case of chunked request body, the rest of the current chunk to
discard).
Similarly, the $content_length variable uses r->headers_in.content_length
if available, and also incorrect. The $content_length variable is used
when proxying with fastcgi_pass, grpc_pass, and uwsgi_pass (scgi_pass uses
the value calculated based on the actual request body buffers, and therefore
works correctly).
diff --git a/body_discard.t b/body_discard.t
new file mode 100644
--- /dev/null
+++ b/body_discard.t
@@ -0,0 +1,449 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for discarding request body.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()
+ ->has(qw/http proxy rewrite addition memcached/);
+
+plan(skip_all => 'not yet') unless $t->has_version('1.25.5');
+
+$t->plan(33)->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+ %%TEST_GLOBALS_HTTP%%
+
+ server {
+ listen 127.0.0.1:8080;
+ server_name localhost;
+
+ lingering_timeout 1s;
+ add_header X-Body body:$content_length:$request_body:;
+
+ client_max_body_size 1k;
+
+ error_page 400 /proxy/error400;
+
+ location / {
+ error_page 413 /error413;
+ proxy_pass http://127.0.0.1:8082;
+ }
+
+ location /error413 {
+ return 200 "custom error 413";
+ }
+
+ location /add {
+ return 200 "main response";
+ add_before_body /add/before;
+ addition_types *;
+ client_max_body_size 1m;
+ }
+
+ location /add/before {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /memcached {
+ client_max_body_size 1m;
+ error_page 502 /memcached/error502;
+ memcached_pass 127.0.0.1:8083;
+ set $memcached_key $request_uri;
+ }
+
+ location /memcached/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy {
+ client_max_body_size 1;
+ error_page 413 /proxy/error413;
+ error_page 400 /proxy/error400;
+ error_page 502 /proxy/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /proxy/error413 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error400 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /unbuf {
+ client_max_body_size 1m;
+ error_page 502 /unbuf/error502;
+ proxy_pass http://127.0.0.1:8083;
+ proxy_request_buffering off;
+ proxy_http_version 1.1;
+ }
+
+ location /unbuf/error502 {
+ client_max_body_size 1m;
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /length {
+ client_max_body_size 1;
+ error_page 413 /length/error413;
+ error_page 502 /length/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /length/error413 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+
+ location /length/error502 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8081;
+ server_name localhost;
+
+ location / {
+ proxy_pass http://127.0.0.1:8082;
+ proxy_set_header X-Body body:$content_length:$request_body:;
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8082;
+ server_name localhost;
+
+ return 200 "backend $http_x_body";
+ }
+
+ server {
+ listen 127.0.0.1:8083;
+ server_name localhost;
+
+ return 444;
+ }
+}
+
+EOF
+
+$t->run();
+
+###############################################################################
+
+# error_page 413 should work without redefining client_max_body_size
+
+like(http(
+ 'POST / HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/ 413 .*custom error 413/s, 'custom error 413');
+
+# subrequest after discarding body
+
+like(http(
+ 'GET /add HTTP/1.0' . CRLF . CRLF
+), qr/backend body:::.*main response/s, 'add');
+
+like(http(
+ 'POST /add HTTP/1.0' . CRLF .
+ 'Content-Length: 10' . CRLF . CRLF .
+ '0123456789'
+), qr/backend body:::.*main response/s, 'add small');
+
+like(http(
+ 'POST /add HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/backend body:::.*main response/s, 'add long');
+
+like(http(
+ 'POST /add HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'a' . CRLF .
+ '0123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/backend body:::.*main response/s, 'add chunked');
+
+like(http(
+ 'POST /add HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF .
+ '9' . CRLF .
+ '123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/backend body:::.*main response/s, 'add chunked multi');
+
+like(http(
+ 'POST /add HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'ffff' . CRLF .
+ '0123456789'
+), qr/backend body:::.*main response/s, 'add chunked long');
+
+# error_page 502 with proxy_pass after discarding body
+
+like(http(
+ 'GET /memcached HTTP/1.0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'memcached');
+
+like(http(
+ 'GET /memcached HTTP/1.0' . CRLF .
+ 'Content-Length: 10' . CRLF . CRLF .
+ '0123456789'
+), qr/ 502 .*backend body:::/s, 'memcached small');
+
+like(http(
+ 'GET /memcached HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/ 502 .*backend body:::/s, 'memcached long');
+
+like(http(
+ 'GET /memcached HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'a' . CRLF .
+ '0123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'memcached chunked');
+
+like(http(
+ 'GET /memcached HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF .
+ '9' . CRLF .
+ '123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'memcached chunked multi');
+
+like(http(
+ 'GET /memcached HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'ffff' . CRLF .
+ '0123456789'
+), qr/ 502 .*backend body:::/s, 'memcached chunked long');
+
+# error_page 413 with proxy_pass
+
+like(http(
+ 'GET /proxy HTTP/1.0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'proxy');
+
+like(http(
+ 'POST /proxy HTTP/1.0' . CRLF .
+ 'Content-Length: 10' . CRLF . CRLF .
+ '0123456789'
+), qr/ 413 .*backend body:::/s, 'proxy small');
+
+like(http(
+ 'POST /proxy HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/ 413 .*backend body:::/s, 'proxy long');
+
+like(http(
+ 'POST /proxy HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'a' . CRLF .
+ '0123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 413 .*backend body:::/s, 'proxy chunked');
+
+like(http(
+ 'POST /proxy HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF .
+ '9' . CRLF .
+ '123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 413 .*backend body:::/s, 'proxy chunked multi');
+
+like(http(
+ 'POST /proxy HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'ffff' . CRLF .
+ '0123456789'
+), qr/ 413 .*backend body:::/s, 'proxy chunked long');
+
+# error_page 400 with proxy_pass
+
+# note that "chunked and length" test triggers 400 during parsing
+# request headers, and therefore needs error_page at server level
+
+like(http(
+ 'POST /proxy HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF .
+ 'X' . CRLF
+), qr/ 400 .*backend body:::/s, 'proxy chunked bad');
+
+like(http(
+ 'POST /proxy HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Content-Length: 10' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 400 .*backend body:::/s, 'proxy chunked and length');
+
+# error_page 502 after proxy with request buffering disabled
+
+like(http(
+ 'GET /unbuf HTTP/1.0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'unbuf proxy');
+
+like(http(
+ 'POST /unbuf HTTP/1.0' . CRLF .
+ 'Content-Length: 10' . CRLF . CRLF .
+ '0',
+ sleep => 0.1,
+ body =>
+ '123456789'
+), qr/ 502 .*backend body:::/s, 'unbuf proxy small');
+
+like(http(
+ 'POST /unbuf HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/ 502 .*backend body:::/s, 'unbuf proxy long');
+
+like(http(
+ 'POST /unbuf HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF,
+ sleep => 0.1,
+ body =>
+ 'a' . CRLF .
+ '0123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'unbuf proxy chunked');
+
+like(http(
+ 'POST /unbuf HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF,
+ sleep => 0.1,
+ body =>
+ '9' . CRLF .
+ '123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 502 .*backend body:::/s, 'unbuf proxy chunked multi');
+
+like(http(
+ 'POST /unbuf HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'ffff' . CRLF .
+ '0123456789'
+), qr/ 502 .*backend body:::/s, 'unbuf proxy chunked long');
+
+# error_page 413 and $content_length
+# (used in fastcgi_pass, grpc_pass, uwsgi_pass)
+
+like(http(
+ 'GET /length HTTP/1.0' . CRLF . CRLF
+), qr/ 502 .*frontend body:::/s, '$content_length');
+
+like(http(
+ 'POST /length HTTP/1.0' . CRLF .
+ 'Content-Length: 10' . CRLF . CRLF .
+ '0123456789'
+), qr/ 413 .*frontend body:::/s, '$content_length small');
+
+like(http(
+ 'POST /length HTTP/1.0' . CRLF .
+ 'Content-Length: 10000' . CRLF . CRLF .
+ '0123456789'
+), qr/ 413 .*frontend body:::/s, '$content_length long');
+
+like(http(
+ 'POST /length HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'a' . CRLF .
+ '0123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 413 .*frontend body:::/s, '$content_length chunked');
+
+like(http(
+ 'POST /length HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ '1' . CRLF .
+ 'X' . CRLF .
+ '9' . CRLF .
+ '123456789' . CRLF .
+ '0' . CRLF . CRLF
+), qr/ 413 .*frontend body:::/s, '$content_length chunked multi');
+
+like(http(
+ 'POST /length HTTP/1.1' . CRLF .
+ 'Host: localhost' . CRLF .
+ 'Connection: close' . CRLF .
+ 'Transfer-Encoding: chunked' . CRLF . CRLF .
+ 'ffff' . CRLF .
+ '0123456789'
+), qr/ 413 .*frontend body:::/s, '$content_length chunked long');
+
+###############################################################################
diff --git a/h2_request_body_discard.t b/h2_request_body_discard.t
new file mode 100644
--- /dev/null
+++ b/h2_request_body_discard.t
@@ -0,0 +1,401 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for discarding request body with HTTP/2.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+use Test::Nginx::HTTP2;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()
+ ->has(qw/http http_v2 proxy rewrite addition/);
+
+plan(skip_all => 'not yet') unless $t->has_version('1.25.5');
+
+$t->plan(34)->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+ %%TEST_GLOBALS_HTTP%%
+
+ server {
+ listen 127.0.0.1:8080;
+ server_name localhost;
+
+ http2 on;
+
+ lingering_timeout 1s;
+ add_header X-Body body:$content_length:$request_body:;
+
+ client_max_body_size 1k;
+
+ error_page 400 /proxy/error400;
+
+ location / {
+ error_page 413 /error413;
+ proxy_pass http://127.0.0.1:8082;
+ }
+
+ location /error413 {
+ return 200 "custom error 413";
+ }
+
+ location /add {
+ return 200 "main response";
+ add_before_body /add/before;
+ addition_types *;
+ client_max_body_size 1m;
+ }
+
+ location /add/before {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /memcached {
+ client_max_body_size 1m;
+ error_page 502 /memcached/error502;
+ memcached_pass 127.0.0.1:8083;
+ set $memcached_key $request_uri;
+ }
+
+ location /memcached/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy {
+ client_max_body_size 3;
+ error_page 413 /proxy/error413;
+ error_page 400 /proxy/error400;
+ error_page 502 /proxy/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /proxy/error413 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error400 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /unbuf {
+ client_max_body_size 1m;
+ error_page 502 /unbuf/error502;
+ proxy_pass http://127.0.0.1:8083;
+ proxy_request_buffering off;
+ proxy_http_version 1.1;
+ }
+
+ location /unbuf/error502 {
+ client_max_body_size 1m;
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /length {
+ client_max_body_size 1;
+ error_page 413 /length/error413;
+ error_page 502 /length/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /length/error413 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+
+ location /length/error502 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8081;
+ server_name localhost;
+
+ location / {
+ proxy_pass http://127.0.0.1:8082;
+ proxy_set_header X-Body body:$content_length:$request_body:;
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8082;
+ server_name localhost;
+
+ return 200 "backend $http_x_body";
+ }
+
+ server {
+ listen 127.0.0.1:8083;
+ server_name localhost;
+
+ return 444;
+ }
+}
+
+EOF
+
+$t->run();
+
+###############################################################################
+
+# error_page 413 should work without redefining client_max_body_size
+
+like(http2_get_body('/', '0123456789' x 128),
+ qr/status: 413.*custom error 413/s, 'custom error 413');
+
+# subrequest after discarding body
+
+like(http2_get('/add'), qr/backend body:::.*main response/s, 'add');
+
+like(http2_get_body('/add', '0123456789'),
+ qr/backend body:::.*main response/s, 'add small');
+
+like(http2_get_body_incomplete('/add', 10000, '0123456789'),
+ qr/backend body:::.*main response/s, 'add long');
+
+like(http2_get_body_nolen('/add', '0123456789'),
+ qr/backend body:::.*main response/s, 'add nolen');
+
+like(http2_get_body_nolen('/add', '0', '123456789'),
+ qr/backend body:::.*main response/s, 'add nolen multi');
+
+like(http2_get_body_incomplete_nolen('/add', 10000, '0123456789'),
+ qr/backend body:::.*main response/s, 'add chunked long');
+
+# error_page 502 with proxy_pass after discarding body
+
+like(http2_get('/memcached'), qr/backend body:::/s, 'memcached');
+
+like(http2_get_body('/memcached', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached small');
+
+like(http2_get_body_incomplete('/memcached', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached long');
+
+like(http2_get_body_nolen('/memcached', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen');
+
+like(http2_get_body_nolen('/memcached', '0', '123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen multi');
+
+like(http2_get_body_incomplete_nolen('/memcached', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen long');
+
+# error_page 413 with proxy_pass
+
+like(http2_get('/proxy'),
+ qr/status: 502.*backend body:::/s, 'proxy');
+
+like(http2_get_body('/proxy', '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy small');
+
+like(http2_get_body_incomplete('/proxy', 10000, '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy long');
+
+like(http2_get_body_nolen('/proxy', '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen');
+
+like(http2_get_body_nolen('/proxy', '0', '123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen multi');
+
+like(http2_get_body_incomplete_nolen('/proxy', 10000, '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen long');
+
+# error_page 400 with proxy_pass
+
+# note that "proxy too short" test triggers 400 during parsing
+# request headers, and therefore needs error_page at server level
+
+like(http2_get_body_custom('/proxy', 1),
+ qr/status: 400.*backend body:::/s, 'proxy too short');
+like(http2_get_body_custom('/proxy', 1, ''),
+ qr/status: 400.*backend body:::/s, 'proxy too short body');
+like(http2_get_body_custom('/proxy', 1, '01'),
+ qr/status: 400.*backend body:::/s, 'proxy too long');
+
+# error_page 502 after proxy with request buffering disabled
+
+like(http2_get('/unbuf'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy');
+
+like(http2_get_body('/unbuf', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy small');
+
+like(http2_get_body_incomplete('/unbuf', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy long');
+
+like(http2_get_body_nolen('/unbuf', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen');
+
+like(http2_get_body_nolen('/unbuf', '0', '123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen multi');
+
+like(http2_get_body_incomplete_nolen('/unbuf', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen long');
+
+# error_page 413 and $content_length
+# (used in fastcgi_pass, grpc_pass, uwsgi_pass)
+
+like(http2_get('/length'),
+ qr/status: 502.*frontend body:::/s, '$content_length');
+
+like(http2_get_body('/length', '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length small');
+
+like(http2_get_body_incomplete('/length', 10000, '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length long');
+
+like(http2_get_body_nolen('/length', '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen');
+
+like(http2_get_body_nolen('/length', '0', '123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen multi');
+
+like(http2_get_body_incomplete_nolen('/length', 10000, '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen long');
+
+###############################################################################
+
+sub http2_get {
+ my ($uri) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({ path => $uri });
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+sub http2_get_body {
+ my ($uri, $body) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({ path => $uri, body => $body });
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+sub http2_get_body_nolen {
+ my ($uri, $body, $body2) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({ path => $uri, body_more => 1 });
+
+ if (defined $body2) {
+ $s->h2_body($body, { body_more => 1 });
+ $s->h2_body($body2);
+ } else {
+ $s->h2_body($body);
+ }
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+sub http2_get_body_incomplete {
+ my ($uri, $len, $body) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({
+ headers => [
+ { name => ':method', value => 'GET' },
+ { name => ':scheme', value => 'http' },
+ { name => ':path', value => $uri },
+ { name => ':authority', value => 'localhost' },
+ { name => 'content-length', value => $len },
+ ],
+ body_more => 1
+ });
+ $s->h2_body($body, { body_more => 1 });
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+sub http2_get_body_incomplete_nolen {
+ my ($uri, $len, $body) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({ path => $uri, body_more => 1 });
+ $s->h2_body($body, { body_more => 1 });
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+sub http2_get_body_custom {
+ my ($uri, $len, $body) = @_;
+
+ my $s = Test::Nginx::HTTP2->new();
+ my $sid = $s->new_stream({
+ headers => [
+ { name => ':method', value => 'GET' },
+ { name => ':scheme', value => 'http' },
+ { name => ':path', value => $uri },
+ { name => ':authority', value => 'localhost' },
+ { name => 'content-length', value => $len },
+ ],
+ body_more => (defined $body ? 1 : undef)
+ });
+ $s->h2_body($body) if defined $body;
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my ($data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n" . $data->{data};
+}
+
+###############################################################################
diff --git a/h3_request_body_discard.t b/h3_request_body_discard.t
new file mode 100644
--- /dev/null
+++ b/h3_request_body_discard.t
@@ -0,0 +1,424 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for discarding request body with HTTP/3.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+use Test::Nginx::HTTP3;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()
+ ->has(qw/http http_v3 proxy rewrite addition memcached cryptx/)
+ ->has_daemon('openssl');
+
+plan(skip_all => 'not yet') unless $t->has_version('1.25.5');
+
+$t->plan(33)->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+ %%TEST_GLOBALS_HTTP%%
+
+ ssl_certificate localhost.crt;
+ ssl_certificate_key localhost.key;
+
+ server {
+ listen 127.0.0.1:%%PORT_8980_UDP%% quic;
+ server_name localhost;
+
+ lingering_timeout 1s;
+ add_header X-Body body:$content_length:$request_body:;
+
+ client_max_body_size 1k;
+
+ location / {
+ error_page 413 /error413;
+ proxy_pass http://127.0.0.1:8082;
+ }
+
+ location /error413 {
+ return 200 "custom error 413";
+ }
+
+ location /add {
+ return 200 "main response";
+ add_before_body /add/before;
+ addition_types *;
+ client_max_body_size 1m;
+ }
+
+ location /add/before {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /memcached {
+ client_max_body_size 1m;
+ error_page 502 /memcached/error502;
+ memcached_pass 127.0.0.1:8083;
+ set $memcached_key $request_uri;
+ }
+
+ location /memcached/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy {
+ client_max_body_size 3;
+ error_page 413 /proxy/error413;
+ error_page 400 /proxy/error400;
+ error_page 502 /proxy/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /proxy/error413 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error400 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /proxy/error502 {
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /unbuf {
+ client_max_body_size 1m;
+ error_page 502 /unbuf/error502;
+ proxy_pass http://127.0.0.1:8083;
+ proxy_request_buffering off;
+ proxy_http_version 1.1;
+ }
+
+ location /unbuf/error502 {
+ client_max_body_size 1m;
+ proxy_pass http://127.0.0.1:8081;
+ }
+
+ location /length {
+ client_max_body_size 1;
+ error_page 413 /length/error413;
+ error_page 502 /length/error502;
+ proxy_pass http://127.0.0.1:8083;
+ }
+
+ location /length/error413 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+
+ location /length/error502 {
+ return 200 "frontend body:$content_length:$request_body:";
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8081;
+ server_name localhost;
+
+ location / {
+ proxy_pass http://127.0.0.1:8082;
+ proxy_set_header X-Body body:$content_length:$request_body:;
+ }
+ }
+
+ server {
+ listen 127.0.0.1:8082;
+ server_name localhost;
+
+ return 200 "backend $http_x_body";
+ }
+
+ server {
+ listen 127.0.0.1:8083;
+ server_name localhost;
+
+ return 444;
+ }
+}
+
+EOF
+
+$t->write_file('openssl.conf', <<EOF);
+[ req ]
+default_bits = 2048
+encrypt_key = no
+distinguished_name = req_distinguished_name
+[ req_distinguished_name ]
+EOF
+
+my $d = $t->testdir();
+
+foreach my $name ('localhost') {
+ system('openssl req -x509 -new '
+ . "-config $d/openssl.conf -subj /CN=$name/ "
+ . "-out $d/$name.crt -keyout $d/$name.key "
+ . ">>$d/openssl.out 2>&1") == 0
+ or die "Can't create certificate for $name: $!\n";
+}
+
+$t->run();
+
+###############################################################################
+
+# error_page 413 should work without redefining client_max_body_size
+
+like(http3_get_body('/', '0123456789' x 128),
+ qr/status: 413.*custom error 413/s, 'custom error 413');
+
+# subrequest after discarding body
+
+like(http3_get('/add'), qr/backend body:::.*main response/s, 'add');
+
+like(http3_get_body('/add', '0123456789'),
+ qr/backend body:::.*main response/s, 'add small');
+
+like(http3_get_body_incomplete('/add', 10000, '0123456789'),
+ qr/backend body:::.*main response/s, 'add long');
+
+like(http3_get_body_nolen('/add', '0123456789'),
+ qr/backend body:::.*main response/s, 'add nolen');
+
+like(http3_get_body_nolen('/add', '0', '123456789'),
+ qr/backend body:::.*main response/s, 'add nolen multi');
+
+like(http3_get_body_incomplete_nolen('/add', 10000, '0123456789'),
+ qr/backend body:::.*main response/s, 'add chunked long');
+
+# error_page 502 with proxy_pass after discarding body
+
+like(http3_get('/memcached'), qr/backend body:::/s, 'memcached');
+
+like(http3_get_body('/memcached', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached small');
+
+like(http3_get_body_incomplete('/memcached', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached long');
+
+like(http3_get_body_nolen('/memcached', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen');
+
+like(http3_get_body_nolen('/memcached', '0', '123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen multi');
+
+like(http3_get_body_incomplete_nolen('/memcached', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'memcached nolen long');
+
+# error_page 413 with proxy_pass
+
+like(http3_get('/proxy'),
+ qr/status: 502.*backend body:::/s, 'proxy');
+
+like(http3_get_body('/proxy', '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy small');
+
+like(http3_get_body_incomplete('/proxy', 10000, '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy long');
+
+like(http3_get_body_nolen('/proxy', '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen');
+
+like(http3_get_body_nolen('/proxy', '0', '123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen multi');
+
+like(http3_get_body_incomplete_nolen('/proxy', '0123456789'),
+ qr/status: 413.*backend body:::/s, 'proxy nolen long');
+
+# error_page 400 with proxy_pass
+
+like(http3_get_body_custom('/proxy', 1, ''),
+ qr/status: 400.*backend body:::/s, 'proxy too short');
+like(http3_get_body_custom('/proxy', 1, '01'),
+ qr/status: 400.*backend body:::/s, 'proxy too long');
+
+# error_page 502 after proxy with request buffering disabled
+
+like(http3_get('/unbuf'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy');
+
+like(http3_get_body_custom('/unbuf', 10, '0123456789', sleep => 0.1),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy small');
+
+like(http3_get_body_incomplete('/unbuf', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy long');
+
+like(http3_get_body_nolen('/unbuf', '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen');
+
+like(http3_get_body_nolen('/unbuf', '0', '123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen multi');
+
+like(http3_get_body_incomplete_nolen('/unbuf', 10000, '0123456789'),
+ qr/status: 502.*backend body:::/s, 'unbuf proxy nolen long');
+
+# error_page 413 and $content_length
+# (used in fastcgi_pass, grpc_pass, uwsgi_pass)
+
+like(http3_get('/length'),
+ qr/status: 502.*frontend body:::/s, '$content_length');
+
+like(http3_get_body('/length', '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length small');
+
+like(http3_get_body_incomplete('/length', 10000, '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length long');
+
+like(http3_get_body_nolen('/length', '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen');
+
+like(http3_get_body_nolen('/length', '0', '123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen multi');
+
+like(http3_get_body_incomplete_nolen('/length', 10000, '0123456789'),
+ qr/status: 413.*frontend body:::/s, '$content_length nolen long');
+
+###############################################################################
+
+sub http3_get {
+ my ($uri) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({ path => $uri });
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+sub http3_get_body {
+ my ($uri, $body) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({ path => $uri, body => $body });
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+sub http3_get_body_nolen {
+ my ($uri, $body, $body2) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({ path => $uri, body_more => 1 });
+
+ if (defined $body2) {
+ select undef, undef, undef, 0.1;
+ $s->h3_body($body, $sid, { body_more => 1 });
+ select undef, undef, undef, 0.1;
+ $s->h3_body($body2, $sid);
+ } else {
+ select undef, undef, undef, 0.1;
+ $s->h3_body($body, $sid);
+ }
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+sub http3_get_body_incomplete {
+ my ($uri, $len, $body) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({
+ headers => [
+ { name => ':method', value => 'GET' },
+ { name => ':scheme', value => 'http' },
+ { name => ':path', value => $uri },
+ { name => ':authority', value => 'localhost' },
+ { name => 'content-length', value => $len },
+ ],
+ body_more => 1
+ });
+ $s->h3_body($body, $sid, { body_more => 1 });
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+sub http3_get_body_incomplete_nolen {
+ my ($uri, $body) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({ path => $uri, body_more => 1 });
+ $s->h3_body($body, $sid, { body_more => 1 });
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+sub http3_get_body_custom {
+ my ($uri, $len, $body, %extra) = @_;
+
+ my $s = Test::Nginx::HTTP3->new();
+ my $sid = $s->new_stream({
+ headers => [
+ { name => ':method', value => 'GET' },
+ { name => ':scheme', value => 'http' },
+ { name => ':path', value => $uri },
+ { name => ':authority', value => 'localhost' },
+ { name => 'content-length', value => $len },
+ ],
+ body_more => 1
+ });
+ select undef, undef, undef, $extra{sleep} if $extra{sleep};
+ $s->h3_body($body, $sid);
+
+ my $frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+ my ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+ my (@data) = grep { $_->{type} eq "DATA" } @$frames;
+
+ return join("\n", map { "$_: " . $frame->{headers}->{$_}; }
+ keys %{$frame->{headers}}) . "\n\n"
+ . join("", map { $_->{data} } @data);
+}
+
+###############################################################################
--
Maxim Dounin
http://mdounin.ru/
More information about the nginx-devel
mailing list