From mdounin at mdounin.ru Sat Feb 8 01:17:25 2025 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sat, 08 Feb 2025 04:17:25 +0300 Subject: [PATCH] Gzip: compatibility with zlib in LIT_MEM mode Message-ID: <18f61fc6a6c092680e98.1738977445@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1738536371 -10800 # Mon Feb 03 01:46:11 2025 +0300 # Node ID 18f61fc6a6c092680e98cb7741226c498ad64163 # Parent dbf76fdd109fbbba40a7c5299cc277d180f4bbad Gzip: compatibility with zlib in LIT_MEM mode. As of zlib 1.3.1, with the LIT_MEM define zlib uses an additional buffer for literals/lengths, that is, (1 << (memlevel + 6)) more memory in the pending buffer. Notably, the LIT_MEM define is enabled by default in zlib-ng. diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -57,6 +57,7 @@ typedef struct { unsigned nomem:1; unsigned buffering:1; unsigned zlib_ng:1; + unsigned zlib_lit_mem:1; unsigned state_allocated:1; size_t zin; @@ -215,6 +216,7 @@ static ngx_http_output_header_filter_pt static ngx_http_output_body_filter_pt ngx_http_next_body_filter; static ngx_uint_t ngx_http_gzip_assume_zlib_ng; +static ngx_uint_t ngx_http_gzip_assume_zlib_lit_mem; static ngx_int_t @@ -512,13 +514,23 @@ ngx_http_gzip_filter_memory(ngx_http_req ctx->allocated = 8192 + 16 + (1 << (wbits + 2)) + (1 << (memlevel + 9)); + if (ngx_http_gzip_assume_zlib_lit_mem) { + /* + * When compiled with the LIT_MEM define, zlib uses an additional + * buffer for literals/lengths. + */ + + ctx->allocated += (1 << (memlevel + 6)); + ctx->zlib_lit_mem = 1; + } + } else { /* * Another zlib variant, https://github.com/zlib-ng/zlib-ng. * It used to force window bits to 13 for fast compression level, * uses (64 + sizeof(void*)) additional space on all allocations * for alignment, 16-byte padding in one of window-sized buffers, - * and 128K hash. + * 128K hash, and defines LIT_MEM by default. */ if (conf->level == 1) { @@ -527,6 +539,7 @@ ngx_http_gzip_filter_memory(ngx_http_req ctx->allocated = 8192 + 16 + (1 << (wbits + 2)) + 131072 + (1 << (memlevel + 8)) + + (1 << (memlevel + 6)) + 4 * (64 + sizeof(void*)); ctx->zlib_ng = 1; } @@ -959,8 +972,11 @@ ngx_http_gzip_filter_alloc(void *opaque, "gzip filter failed to use preallocated memory: " "%ud of %ui", items * size, ctx->allocated); + } else if (ctx->zlib_lit_mem) { + ngx_http_gzip_assume_zlib_ng = 1; + } else { - ngx_http_gzip_assume_zlib_ng = 1; + ngx_http_gzip_assume_zlib_lit_mem = 1; } p = ngx_palloc(ctx->request->pool, items * size); From mdounin at mdounin.ru Sat Feb 8 01:19:59 2025 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sat, 08 Feb 2025 04:19:59 +0300 Subject: [PATCH] Tests: gzip preallocation tests Message-ID: # HG changeset patch # User Maxim Dounin # Date 1738636692 -10800 # Tue Feb 04 05:38:12 2025 +0300 # Node ID a848cb246f6cf33448d78c8596ae1e7cfcb6f042 # Parent 15f538440a7734a57353054f2f2e33808f8a1174 Tests: gzip preallocation tests. diff --git a/gzip_alloc.t b/gzip_alloc.t new file mode 100644 --- /dev/null +++ b/gzip_alloc.t @@ -0,0 +1,169 @@ +#!/usr/bin/perl + +# (C) Maxim Dounin + +# Tests for nginx gzip filter module, preallocation sizes with various +# settings. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx qw/ :DEFAULT :gzip /; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http gzip/)->plan(50); + +$t->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + gzip on; + } + + location /max/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 32k; + gzip_hash 128k; + } + + location /min/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 512; + gzip_hash 512; + } + + location /minw/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 512; + gzip_hash 128k; + } + + location /minh/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 32k; + gzip_hash 512; + } + } +} + +EOF + +$t->write_file('64.html', 'X' x 64); +$t->write_file('512.html', 'X' x 512); +$t->write_file('1k.html', 'X' x 1024); +$t->write_file('2k.html', 'X' x 2048); +$t->write_file('4k.html', 'X' x 4096); +$t->write_file('8k.html', 'X' x 8196); +$t->write_file('16k.html', 'X' x 16384); +$t->write_file('32k.html', 'X' x 32768); +$t->write_file('64k.html', 'X' x 65536); +$t->write_file('128k.html', 'X' x 131072); + +$t->run(); + +############################################################################### + +http_gzip_like(http_gzip_request('/64.html'), qr/^X{64}\Z/, 'gzip 64'); +http_gzip_like(http_gzip_request('/512.html'), qr/^X{512}\Z/, 'gzip 512'); +http_gzip_like(http_gzip_request('/1k.html'), qr/^X{1024}\Z/, 'gzip 1k'); +http_gzip_like(http_gzip_request('/2k.html'), qr/^X{2048}\Z/, 'gzip 2k'); +http_gzip_like(http_gzip_request('/4k.html'), qr/^X{2048}X+\Z/, 'gzip 4k'); +http_gzip_like(http_gzip_request('/8k.html'), qr/^X{2048}X+\Z/, 'gzip 8k'); +http_gzip_like(http_gzip_request('/16k.html'), qr/^X{2048}X+\Z/, 'gzip 16k'); +http_gzip_like(http_gzip_request('/32k.html'), qr/^X{2048}X+\Z/, 'gzip 32k'); +http_gzip_like(http_gzip_request('/64k.html'), qr/^X{2048}X+\Z/, 'gzip 64k'); +http_gzip_like(http_gzip_request('/128k.html'), qr/^X{2048}X+\Z/, 'gzip 128k'); + +http_gzip_like(http_gzip_request('/max/64.html'), qr/^X+\Z/, 'gzip max 64'); +http_gzip_like(http_gzip_request('/max/512.html'), qr/^X+\Z/, 'gzip max 512'); +http_gzip_like(http_gzip_request('/max/1k.html'), qr/^X+\Z/, 'gzip max 1k'); +http_gzip_like(http_gzip_request('/max/2k.html'), qr/^X+\Z/, 'gzip max 2k'); +http_gzip_like(http_gzip_request('/max/4k.html'), qr/^X+\Z/, 'gzip max 4k'); +http_gzip_like(http_gzip_request('/max/8k.html'), qr/^X+\Z/, 'gzip max 8k'); +http_gzip_like(http_gzip_request('/max/16k.html'), qr/^X+\Z/, 'gzip max 16k'); +http_gzip_like(http_gzip_request('/max/32k.html'), qr/^X+\Z/, 'gzip max 32k'); +http_gzip_like(http_gzip_request('/max/64k.html'), qr/^X+\Z/, 'gzip max 64k'); +http_gzip_like(http_gzip_request('/max/128k.html'), qr/^X+\Z/, 'gzip max 128k'); + +http_gzip_like(http_gzip_request('/min/64.html'), qr/^X+\Z/, 'gzip min 64'); +http_gzip_like(http_gzip_request('/max/512.html'), qr/^X+\Z/, 'gzip min 512'); +http_gzip_like(http_gzip_request('/min/1k.html'), qr/^X+\Z/, 'gzip min 1k'); +http_gzip_like(http_gzip_request('/min/2k.html'), qr/^X+\Z/, 'gzip min 2k'); +http_gzip_like(http_gzip_request('/min/4k.html'), qr/^X+\Z/, 'gzip min 4k'); +http_gzip_like(http_gzip_request('/min/8k.html'), qr/^X+\Z/, 'gzip min 8k'); +http_gzip_like(http_gzip_request('/min/16k.html'), qr/^X+\Z/, 'gzip min 16k'); +http_gzip_like(http_gzip_request('/min/32k.html'), qr/^X+\Z/, 'gzip min 32k'); +http_gzip_like(http_gzip_request('/min/64k.html'), qr/^X+\Z/, 'gzip min 64k'); +http_gzip_like(http_gzip_request('/min/128k.html'), qr/^X+\Z/, 'gzip min 128k'); + +http_gzip_like(http_gzip_request('/minw/64.html'), qr/^X+\Z/, + 'gzip min window 64'); +http_gzip_like(http_gzip_request('/minw/512.html'), qr/^X+\Z/, + 'gzip min window 512'); +http_gzip_like(http_gzip_request('/minw/1k.html'), qr/^X+\Z/, + 'gzip min window 1k'); +http_gzip_like(http_gzip_request('/minw/2k.html'), qr/^X+\Z/, + 'gzip min window 2k'); +http_gzip_like(http_gzip_request('/minw/4k.html'), qr/^X+\Z/, + 'gzip min window 4k'); +http_gzip_like(http_gzip_request('/minw/8k.html'), qr/^X+\Z/, + 'gzip min window 8k'); +http_gzip_like(http_gzip_request('/minw/16k.html'), qr/^X+\Z/, + 'gzip min window 16k'); +http_gzip_like(http_gzip_request('/minw/32k.html'), qr/^X+\Z/, + 'gzip min window 32k'); +http_gzip_like(http_gzip_request('/minw/64k.html'), qr/^X+\Z/, + 'gzip min window 64k'); +http_gzip_like(http_gzip_request('/minw/128k.html'), qr/^X+\Z/, + 'gzip min window 128k'); + +http_gzip_like(http_gzip_request('/minh/64.html'), qr/^X+\Z/, + 'gzip min hash 64'); +http_gzip_like(http_gzip_request('/minh/512.html'), qr/^X+\Z/, + 'gzip min hash 512'); +http_gzip_like(http_gzip_request('/minh/1k.html'), qr/^X+\Z/, + 'gzip min hash 1k'); +http_gzip_like(http_gzip_request('/minh/2k.html'), qr/^X+\Z/, + 'gzip min hash 2k'); +http_gzip_like(http_gzip_request('/minh/4k.html'), qr/^X+\Z/, + 'gzip min hash 4k'); +http_gzip_like(http_gzip_request('/minh/8k.html'), qr/^X+\Z/, + 'gzip min hash 8k'); +http_gzip_like(http_gzip_request('/minh/16k.html'), qr/^X+\Z/, + 'gzip min hash 16k'); +http_gzip_like(http_gzip_request('/minh/32k.html'), qr/^X+\Z/, + 'gzip min hash 32k'); +http_gzip_like(http_gzip_request('/minh/64k.html'), qr/^X+\Z/, + 'gzip min hash 64k'); +http_gzip_like(http_gzip_request('/minh/128k.html'), qr/^X+\Z/, + 'gzip min hash 128k'); + +############################################################################### From mdounin at mdounin.ru Sat Feb 8 01:22:12 2025 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sat, 08 Feb 2025 04:22:12 +0300 Subject: [PATCH 1 of 3] Tests: fixed "ffmpeg -nostdin" usage in range_mp4.t Message-ID: <4b0715cba7825dac5bc6.1738977732@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1738716192 -10800 # Wed Feb 05 03:43:12 2025 +0300 # Node ID 4b0715cba7825dac5bc682b16f9acf1b664aab60 # Parent a848cb246f6cf33448d78c8596ae1e7cfcb6f042 Tests: fixed "ffmpeg -nostdin" usage in range_mp4.t. Usage of "ffmpeg -formats" do not require "-nostdin", as terminal is only touched after parsing options, and "-formats" will cause an early exit during parsing. Contrary, test mp4 file generation needs "-nostdin". This is how it is used in other mp4 tests, but in range_mp4.t it was somehow mixed up in 1268:1923461981c9. diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -48,8 +48,8 @@ http { EOF plan(skip_all => 'no lavfi') - unless grep /lavfi/, `ffmpeg -nostdin -loglevel quiet -formats`; -system('ffmpeg -loglevel quiet -y ' + unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; From mdounin at mdounin.ru Sat Feb 8 01:22:13 2025 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sat, 08 Feb 2025 04:22:13 +0300 Subject: [PATCH 2 of 3] Tests: adjusted mp4 tests to check if libx264 encoder available In-Reply-To: <4b0715cba7825dac5bc6.1738977732@vm-bsd.mdounin.ru> References: <4b0715cba7825dac5bc6.1738977732@vm-bsd.mdounin.ru> Message-ID: # HG changeset patch # User Maxim Dounin # Date 1738720100 -10800 # Wed Feb 05 04:48:20 2025 +0300 # Node ID fd59bcf3fca87d69b369c1e637038dac132ee73c # Parent 4b0715cba7825dac5bc682b16f9acf1b664aab60 Tests: adjusted mp4 tests to check if libx264 encoder available. Notably, this fixes tests on Fedora, where libx264 is not available by default. diff --git a/mp4.t b/mp4.t --- a/mp4.t +++ b/mp4.t @@ -51,6 +51,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' diff --git a/mp4_ssi.t b/mp4_ssi.t --- a/mp4_ssi.t +++ b/mp4_ssi.t @@ -52,6 +52,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' diff --git a/mp4_start_key_frame.t b/mp4_start_key_frame.t --- a/mp4_start_key_frame.t +++ b/mp4_start_key_frame.t @@ -56,6 +56,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-pix_fmt yuv420p -g 15 -c:v libx264 ' diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -49,6 +49,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 From mdounin at mdounin.ru Sat Feb 8 01:22:14 2025 From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=) Date: Sat, 08 Feb 2025 04:22:14 +0300 Subject: [PATCH 3 of 3] Tests: adjusted mp4 tests to use libopenh264 encoder if available In-Reply-To: <4b0715cba7825dac5bc6.1738977732@vm-bsd.mdounin.ru> References: <4b0715cba7825dac5bc6.1738977732@vm-bsd.mdounin.ru> Message-ID: <570d0c2667b8bce8d76e.1738977734@vm-bsd.mdounin.ru> # HG changeset patch # User Maxim Dounin # Date 1738891503 -10800 # Fri Feb 07 04:25:03 2025 +0300 # Node ID 570d0c2667b8bce8d76e9f58ddc2515b36d6b0de # Parent fd59bcf3fca87d69b369c1e637038dac132ee73c Tests: adjusted mp4 tests to use libopenh264 encoder if available. Instead of specifically requesting libx264, the h264 codec is now requested, so the default encoder for the codec will be used. Notably, this ensures that tests will properly work on Fedora, where libopenh264 is used instead of libx264. diff --git a/mp4.t b/mp4.t --- a/mp4.t +++ b/mp4.t @@ -51,18 +51,18 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . '-movflags +faststart ' . "${\($t->testdir())}/no_mdat.mp4") == 0 or die "Can't create mp4 file: $!"; diff --git a/mp4_ssi.t b/mp4_ssi.t --- a/mp4_ssi.t +++ b/mp4_ssi.t @@ -52,12 +52,12 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/ssi.mp4") == 0 or die "Can't create mp4 file: $!"; diff --git a/mp4_start_key_frame.t b/mp4_start_key_frame.t --- a/mp4_start_key_frame.t +++ b/mp4_start_key_frame.t @@ -56,11 +56,11 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' - . '-pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; $t->run()->plan(4); diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -49,11 +49,11 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' - . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 + . "-pix_fmt yuv420p -c:v h264 ${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; $t->run()->plan(13); From kirill at korins.ky Sat Feb 8 18:02:20 2025 From: kirill at korins.ky (Kirill A. Korinsky) Date: Sat, 08 Feb 2025 19:02:20 +0100 Subject: [PATCH] Gzip: compatibility with zlib in LIT_MEM mode In-Reply-To: <18f61fc6a6c092680e98.1738977445@vm-bsd.mdounin.ru> References: <18f61fc6a6c092680e98.1738977445@vm-bsd.mdounin.ru> Message-ID: <87pljsnwlv.wl-kirill@korins.ky> I had tried your patch instead of https://github.com/nginx/nginx/commit/57d54fd922e7ecbebb78598d13adc9df1a4b69c0 It works as advertiesed, and I can't reproduce the issue on Fedora 41. -- wbr, Kirill From mdounin at mdounin.ru Mon Feb 10 06:28:49 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:28:49 +0300 Subject: [nginx] Version bump. Message-ID: details: http://freenginx.org/hg/nginx/rev/f61f063144bc branches: changeset: 9332:f61f063144bc user: Maxim Dounin date: Mon Feb 10 09:10:31 2025 +0300 description: Version bump. diffstat: src/core/nginx.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff --git a/src/core/nginx.h b/src/core/nginx.h --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027004 -#define NGINX_VERSION "1.27.4" +#define nginx_version 1027005 +#define NGINX_VERSION "1.27.5" #define NGINX_NAME "freenginx" #define NGINX_VER NGINX_NAME "/" NGINX_VERSION From mdounin at mdounin.ru Mon Feb 10 06:28:50 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:28:50 +0300 Subject: [nginx] Gzip: compatibility with zlib in LIT_MEM mode. Message-ID: details: http://freenginx.org/hg/nginx/rev/1996ea0bc55d branches: changeset: 9333:1996ea0bc55d user: Maxim Dounin date: Mon Feb 10 09:10:57 2025 +0300 description: Gzip: compatibility with zlib in LIT_MEM mode. As of zlib 1.3.1, with the LIT_MEM define zlib uses an additional buffer for literals/lengths, that is, (1 << (memlevel + 6)) more memory in the pending buffer. Notably, the LIT_MEM define is enabled by default in zlib-ng. diffstat: src/http/modules/ngx_http_gzip_filter_module.c | 20 ++++++++++++++++++-- 1 files changed, 18 insertions(+), 2 deletions(-) diffs (65 lines): diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -57,6 +57,7 @@ typedef struct { unsigned nomem:1; unsigned buffering:1; unsigned zlib_ng:1; + unsigned zlib_lit_mem:1; unsigned state_allocated:1; size_t zin; @@ -215,6 +216,7 @@ static ngx_http_output_header_filter_pt static ngx_http_output_body_filter_pt ngx_http_next_body_filter; static ngx_uint_t ngx_http_gzip_assume_zlib_ng; +static ngx_uint_t ngx_http_gzip_assume_zlib_lit_mem; static ngx_int_t @@ -512,13 +514,23 @@ ngx_http_gzip_filter_memory(ngx_http_req ctx->allocated = 8192 + 16 + (1 << (wbits + 2)) + (1 << (memlevel + 9)); + if (ngx_http_gzip_assume_zlib_lit_mem) { + /* + * When compiled with the LIT_MEM define, zlib uses an additional + * buffer for literals/lengths. + */ + + ctx->allocated += (1 << (memlevel + 6)); + ctx->zlib_lit_mem = 1; + } + } else { /* * Another zlib variant, https://github.com/zlib-ng/zlib-ng. * It used to force window bits to 13 for fast compression level, * uses (64 + sizeof(void*)) additional space on all allocations * for alignment, 16-byte padding in one of window-sized buffers, - * and 128K hash. + * 128K hash, and defines LIT_MEM by default. */ if (conf->level == 1) { @@ -527,6 +539,7 @@ ngx_http_gzip_filter_memory(ngx_http_req ctx->allocated = 8192 + 16 + (1 << (wbits + 2)) + 131072 + (1 << (memlevel + 8)) + + (1 << (memlevel + 6)) + 4 * (64 + sizeof(void*)); ctx->zlib_ng = 1; } @@ -959,8 +972,11 @@ ngx_http_gzip_filter_alloc(void *opaque, "gzip filter failed to use preallocated memory: " "%ud of %ui", items * size, ctx->allocated); + } else if (ctx->zlib_lit_mem) { + ngx_http_gzip_assume_zlib_ng = 1; + } else { - ngx_http_gzip_assume_zlib_ng = 1; + ngx_http_gzip_assume_zlib_lit_mem = 1; } p = ngx_palloc(ctx->request->pool, items * size); From mdounin at mdounin.ru Mon Feb 10 06:29:08 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:29:08 +0300 Subject: [nginx-tests] Tests: gzip preallocation tests. Message-ID: details: http://freenginx.org/hg/nginx-tests/rev/a848cb246f6c branches: changeset: 2000:a848cb246f6c user: Maxim Dounin date: Tue Feb 04 05:38:12 2025 +0300 description: Tests: gzip preallocation tests. diffstat: gzip_alloc.t | 169 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 169 insertions(+), 0 deletions(-) diffs (174 lines): diff --git a/gzip_alloc.t b/gzip_alloc.t new file mode 100644 --- /dev/null +++ b/gzip_alloc.t @@ -0,0 +1,169 @@ +#!/usr/bin/perl + +# (C) Maxim Dounin + +# Tests for nginx gzip filter module, preallocation sizes with various +# settings. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx qw/ :DEFAULT :gzip /; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http gzip/)->plan(50); + +$t->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + gzip on; + } + + location /max/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 32k; + gzip_hash 128k; + } + + location /min/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 512; + gzip_hash 512; + } + + location /minw/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 512; + gzip_hash 128k; + } + + location /minh/ { + alias %%TESTDIR%%/; + gzip on; + gzip_window 32k; + gzip_hash 512; + } + } +} + +EOF + +$t->write_file('64.html', 'X' x 64); +$t->write_file('512.html', 'X' x 512); +$t->write_file('1k.html', 'X' x 1024); +$t->write_file('2k.html', 'X' x 2048); +$t->write_file('4k.html', 'X' x 4096); +$t->write_file('8k.html', 'X' x 8196); +$t->write_file('16k.html', 'X' x 16384); +$t->write_file('32k.html', 'X' x 32768); +$t->write_file('64k.html', 'X' x 65536); +$t->write_file('128k.html', 'X' x 131072); + +$t->run(); + +############################################################################### + +http_gzip_like(http_gzip_request('/64.html'), qr/^X{64}\Z/, 'gzip 64'); +http_gzip_like(http_gzip_request('/512.html'), qr/^X{512}\Z/, 'gzip 512'); +http_gzip_like(http_gzip_request('/1k.html'), qr/^X{1024}\Z/, 'gzip 1k'); +http_gzip_like(http_gzip_request('/2k.html'), qr/^X{2048}\Z/, 'gzip 2k'); +http_gzip_like(http_gzip_request('/4k.html'), qr/^X{2048}X+\Z/, 'gzip 4k'); +http_gzip_like(http_gzip_request('/8k.html'), qr/^X{2048}X+\Z/, 'gzip 8k'); +http_gzip_like(http_gzip_request('/16k.html'), qr/^X{2048}X+\Z/, 'gzip 16k'); +http_gzip_like(http_gzip_request('/32k.html'), qr/^X{2048}X+\Z/, 'gzip 32k'); +http_gzip_like(http_gzip_request('/64k.html'), qr/^X{2048}X+\Z/, 'gzip 64k'); +http_gzip_like(http_gzip_request('/128k.html'), qr/^X{2048}X+\Z/, 'gzip 128k'); + +http_gzip_like(http_gzip_request('/max/64.html'), qr/^X+\Z/, 'gzip max 64'); +http_gzip_like(http_gzip_request('/max/512.html'), qr/^X+\Z/, 'gzip max 512'); +http_gzip_like(http_gzip_request('/max/1k.html'), qr/^X+\Z/, 'gzip max 1k'); +http_gzip_like(http_gzip_request('/max/2k.html'), qr/^X+\Z/, 'gzip max 2k'); +http_gzip_like(http_gzip_request('/max/4k.html'), qr/^X+\Z/, 'gzip max 4k'); +http_gzip_like(http_gzip_request('/max/8k.html'), qr/^X+\Z/, 'gzip max 8k'); +http_gzip_like(http_gzip_request('/max/16k.html'), qr/^X+\Z/, 'gzip max 16k'); +http_gzip_like(http_gzip_request('/max/32k.html'), qr/^X+\Z/, 'gzip max 32k'); +http_gzip_like(http_gzip_request('/max/64k.html'), qr/^X+\Z/, 'gzip max 64k'); +http_gzip_like(http_gzip_request('/max/128k.html'), qr/^X+\Z/, 'gzip max 128k'); + +http_gzip_like(http_gzip_request('/min/64.html'), qr/^X+\Z/, 'gzip min 64'); +http_gzip_like(http_gzip_request('/max/512.html'), qr/^X+\Z/, 'gzip min 512'); +http_gzip_like(http_gzip_request('/min/1k.html'), qr/^X+\Z/, 'gzip min 1k'); +http_gzip_like(http_gzip_request('/min/2k.html'), qr/^X+\Z/, 'gzip min 2k'); +http_gzip_like(http_gzip_request('/min/4k.html'), qr/^X+\Z/, 'gzip min 4k'); +http_gzip_like(http_gzip_request('/min/8k.html'), qr/^X+\Z/, 'gzip min 8k'); +http_gzip_like(http_gzip_request('/min/16k.html'), qr/^X+\Z/, 'gzip min 16k'); +http_gzip_like(http_gzip_request('/min/32k.html'), qr/^X+\Z/, 'gzip min 32k'); +http_gzip_like(http_gzip_request('/min/64k.html'), qr/^X+\Z/, 'gzip min 64k'); +http_gzip_like(http_gzip_request('/min/128k.html'), qr/^X+\Z/, 'gzip min 128k'); + +http_gzip_like(http_gzip_request('/minw/64.html'), qr/^X+\Z/, + 'gzip min window 64'); +http_gzip_like(http_gzip_request('/minw/512.html'), qr/^X+\Z/, + 'gzip min window 512'); +http_gzip_like(http_gzip_request('/minw/1k.html'), qr/^X+\Z/, + 'gzip min window 1k'); +http_gzip_like(http_gzip_request('/minw/2k.html'), qr/^X+\Z/, + 'gzip min window 2k'); +http_gzip_like(http_gzip_request('/minw/4k.html'), qr/^X+\Z/, + 'gzip min window 4k'); +http_gzip_like(http_gzip_request('/minw/8k.html'), qr/^X+\Z/, + 'gzip min window 8k'); +http_gzip_like(http_gzip_request('/minw/16k.html'), qr/^X+\Z/, + 'gzip min window 16k'); +http_gzip_like(http_gzip_request('/minw/32k.html'), qr/^X+\Z/, + 'gzip min window 32k'); +http_gzip_like(http_gzip_request('/minw/64k.html'), qr/^X+\Z/, + 'gzip min window 64k'); +http_gzip_like(http_gzip_request('/minw/128k.html'), qr/^X+\Z/, + 'gzip min window 128k'); + +http_gzip_like(http_gzip_request('/minh/64.html'), qr/^X+\Z/, + 'gzip min hash 64'); +http_gzip_like(http_gzip_request('/minh/512.html'), qr/^X+\Z/, + 'gzip min hash 512'); +http_gzip_like(http_gzip_request('/minh/1k.html'), qr/^X+\Z/, + 'gzip min hash 1k'); +http_gzip_like(http_gzip_request('/minh/2k.html'), qr/^X+\Z/, + 'gzip min hash 2k'); +http_gzip_like(http_gzip_request('/minh/4k.html'), qr/^X+\Z/, + 'gzip min hash 4k'); +http_gzip_like(http_gzip_request('/minh/8k.html'), qr/^X+\Z/, + 'gzip min hash 8k'); +http_gzip_like(http_gzip_request('/minh/16k.html'), qr/^X+\Z/, + 'gzip min hash 16k'); +http_gzip_like(http_gzip_request('/minh/32k.html'), qr/^X+\Z/, + 'gzip min hash 32k'); +http_gzip_like(http_gzip_request('/minh/64k.html'), qr/^X+\Z/, + 'gzip min hash 64k'); +http_gzip_like(http_gzip_request('/minh/128k.html'), qr/^X+\Z/, + 'gzip min hash 128k'); + +############################################################################### From mdounin at mdounin.ru Mon Feb 10 06:29:08 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:29:08 +0300 Subject: [nginx-tests] Tests: fixed "ffmpeg -nostdin" usage in range_mp4.t. Message-ID: details: http://freenginx.org/hg/nginx-tests/rev/4b0715cba782 branches: changeset: 2001:4b0715cba782 user: Maxim Dounin date: Wed Feb 05 03:43:12 2025 +0300 description: Tests: fixed "ffmpeg -nostdin" usage in range_mp4.t. Usage of "ffmpeg -formats" do not require "-nostdin", as terminal is only touched after parsing options, and "-formats" will cause an early exit during parsing. Contrary, test mp4 file generation needs "-nostdin". This is how it is used in other mp4 tests, but in range_mp4.t it was somehow mixed up in 1268:1923461981c9. diffstat: range_mp4.t | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -48,8 +48,8 @@ http { EOF plan(skip_all => 'no lavfi') - unless grep /lavfi/, `ffmpeg -nostdin -loglevel quiet -formats`; -system('ffmpeg -loglevel quiet -y ' + unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; From mdounin at mdounin.ru Mon Feb 10 06:29:08 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:29:08 +0300 Subject: [nginx-tests] Tests: adjusted mp4 tests to check if libx264 enco... Message-ID: details: http://freenginx.org/hg/nginx-tests/rev/fd59bcf3fca8 branches: changeset: 2002:fd59bcf3fca8 user: Maxim Dounin date: Wed Feb 05 04:48:20 2025 +0300 description: Tests: adjusted mp4 tests to check if libx264 encoder available. Notably, this fixes tests on Fedora, where libx264 is not available by default. diffstat: mp4.t | 2 ++ mp4_ssi.t | 2 ++ mp4_start_key_frame.t | 2 ++ range_mp4.t | 2 ++ 4 files changed, 8 insertions(+), 0 deletions(-) diffs (48 lines): diff --git a/mp4.t b/mp4.t --- a/mp4.t +++ b/mp4.t @@ -51,6 +51,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' diff --git a/mp4_ssi.t b/mp4_ssi.t --- a/mp4_ssi.t +++ b/mp4_ssi.t @@ -52,6 +52,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' diff --git a/mp4_start_key_frame.t b/mp4_start_key_frame.t --- a/mp4_start_key_frame.t +++ b/mp4_start_key_frame.t @@ -56,6 +56,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-pix_fmt yuv420p -g 15 -c:v libx264 ' diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -49,6 +49,8 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; +plan(skip_all => 'no libx264') + unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 From mdounin at mdounin.ru Mon Feb 10 06:29:08 2025 From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=) Date: Mon, 10 Feb 2025 09:29:08 +0300 Subject: [nginx-tests] Tests: adjusted mp4 tests to use libopenh264 encod... Message-ID: details: http://freenginx.org/hg/nginx-tests/rev/570d0c2667b8 branches: changeset: 2003:570d0c2667b8 user: Maxim Dounin date: Fri Feb 07 04:25:03 2025 +0300 description: Tests: adjusted mp4 tests to use libopenh264 encoder if available. Instead of specifically requesting libx264, the h264 codec is now requested, so the default encoder for the codec will be used. Notably, this ensures that tests will properly work on Fedora, where libopenh264 is used instead of libx264. diffstat: mp4.t | 8 ++++---- mp4_ssi.t | 6 +++--- mp4_start_key_frame.t | 6 +++--- range_mp4.t | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diffs (81 lines): diff --git a/mp4.t b/mp4.t --- a/mp4.t +++ b/mp4.t @@ -51,18 +51,18 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . '-movflags +faststart ' . "${\($t->testdir())}/no_mdat.mp4") == 0 or die "Can't create mp4 file: $!"; diff --git a/mp4_ssi.t b/mp4_ssi.t --- a/mp4_ssi.t +++ b/mp4_ssi.t @@ -52,12 +52,12 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' . '-f lavfi -i testsrc=duration=20:size=320x200:rate=15 ' - . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-map 0:0 -map 1:0 -pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/ssi.mp4") == 0 or die "Can't create mp4 file: $!"; diff --git a/mp4_start_key_frame.t b/mp4_start_key_frame.t --- a/mp4_start_key_frame.t +++ b/mp4_start_key_frame.t @@ -56,11 +56,11 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' - . '-pix_fmt yuv420p -g 15 -c:v libx264 ' + . '-pix_fmt yuv420p -g 15 -c:v h264 ' . "${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; $t->run()->plan(4); diff --git a/range_mp4.t b/range_mp4.t --- a/range_mp4.t +++ b/range_mp4.t @@ -49,11 +49,11 @@ EOF plan(skip_all => 'no lavfi') unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`; -plan(skip_all => 'no libx264') - unless grep /libx264/, `ffmpeg -loglevel quiet -encoders`; +plan(skip_all => 'no libx264 or libopenh264') + unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`; system('ffmpeg -nostdin -loglevel quiet -y ' . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 ' - . "-pix_fmt yuv420p -c:v libx264 ${\($t->testdir())}/test.mp4") == 0 + . "-pix_fmt yuv420p -c:v h264 ${\($t->testdir())}/test.mp4") == 0 or die "Can't create mp4 file: $!"; $t->run()->plan(13); From mdounin at mdounin.ru Mon Feb 10 06:31:26 2025 From: mdounin at mdounin.ru (Maxim Dounin) Date: Mon, 10 Feb 2025 09:31:26 +0300 Subject: [PATCH] Gzip: compatibility with zlib in LIT_MEM mode In-Reply-To: <87pljsnwlv.wl-kirill@korins.ky> References: <18f61fc6a6c092680e98.1738977445@vm-bsd.mdounin.ru> <87pljsnwlv.wl-kirill@korins.ky> Message-ID: Hello! On Sat, Feb 08, 2025 at 07:02:20PM +0100, Kirill A. Korinsky wrote: > I had tried your patch instead of > https://github.com/nginx/nginx/commit/57d54fd922e7ecbebb78598d13adc9df1a4b69c0 > > It works as advertiesed, and I can't reproduce the issue on Fedora 41. Committed, thanks for testing (and prodding this). -- Maxim Dounin http://mdounin.ru/