From mdounin at mdounin.ru  Fri Mar  1 03:18:16 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:16 +0300
Subject: [PATCH 0 of 5] syslog error handling improvements
Message-ID: <patchbomb.1709263096@vm-bsd.mdounin.ru>

Hello!

The following patch series improves error handling when logging
to syslog.  Notably, it introduces error logging moderation: errors
when logging to syslog are now logged at most once per second.

-- 
Maxim Dounin



From mdounin at mdounin.ru  Fri Mar  1 03:18:17 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:17 +0300
Subject: [PATCH 1 of 5] Syslog: fixed duplicate errors with access logging to
 syslog
In-Reply-To: <patchbomb.1709263096@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
Message-ID: <c7c8354f99face52f3b5.1709263097@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1709260929 -10800
#      Fri Mar 01 05:42:09 2024 +0300
# Node ID c7c8354f99face52f3b51442360317f3e2768492
# Parent  697f452bc03301a8fe22d8bcae6f3bc06f4efe6f
Syslog: fixed duplicate errors with access logging to syslog.

The ngx_syslog_send() function logs errors itself, so there is no need
to additionally log errors in the caller, notably access log modules in
http and stream.

To ensure that incomplete and blocked writes are also logged, appropriate
logging added to ngx_syslog_send().

diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -319,6 +319,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         }
 
         peer->conn.fd = (ngx_socket_t) -1;
+
+    } else if ((size_t) n != len) {
+        ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
+                      "send() incomplete");
     }
 
     return n;
diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c
--- a/src/http/modules/ngx_http_log_module.c
+++ b/src/http/modules/ngx_http_log_module.c
@@ -254,8 +254,7 @@ static ngx_int_t
 ngx_http_log_handler(ngx_http_request_t *r)
 {
     u_char                   *line, *p;
-    size_t                    len, size;
-    ssize_t                   n;
+    size_t                    len;
     ngx_str_t                 val;
     ngx_uint_t                i, l;
     ngx_http_log_t           *log;
@@ -376,19 +375,7 @@ ngx_http_log_handler(ngx_http_request_t 
 
         if (log[l].syslog_peer) {
 
-            size = p - line;
-
-            n = ngx_syslog_send(log[l].syslog_peer, line, size);
-
-            if (n < 0) {
-                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                              "send() to syslog failed");
-
-            } else if ((size_t) n != size) {
-                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                              "send() to syslog has written only %z of %uz",
-                              n, size);
-            }
+            (void) ngx_syslog_send(log[l].syslog_peer, line, p - line);
 
             continue;
         }
diff --git a/src/stream/ngx_stream_log_module.c b/src/stream/ngx_stream_log_module.c
--- a/src/stream/ngx_stream_log_module.c
+++ b/src/stream/ngx_stream_log_module.c
@@ -201,8 +201,7 @@ static ngx_int_t
 ngx_stream_log_handler(ngx_stream_session_t *s)
 {
     u_char                     *line, *p;
-    size_t                      len, size;
-    ssize_t                     n;
+    size_t                      len;
     ngx_str_t                   val;
     ngx_uint_t                  i, l;
     ngx_stream_log_t           *log;
@@ -324,19 +323,7 @@ ngx_stream_log_handler(ngx_stream_sessio
 
         if (log[l].syslog_peer) {
 
-            size = p - line;
-
-            n = ngx_syslog_send(log[l].syslog_peer, line, size);
-
-            if (n < 0) {
-                ngx_log_error(NGX_LOG_WARN, s->connection->log, 0,
-                              "send() to syslog failed");
-
-            } else if ((size_t) n != size) {
-                ngx_log_error(NGX_LOG_WARN, s->connection->log, 0,
-                              "send() to syslog has written only %z of %uz",
-                              n, size);
-            }
+            (void) ngx_syslog_send(log[l].syslog_peer, line, p - line);
 
             continue;
         }



From mdounin at mdounin.ru  Fri Mar  1 03:18:18 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:18 +0300
Subject: [PATCH 2 of 5] Syslog: introduced ngx_syslog_send() error logging
 moderation
In-Reply-To: <patchbomb.1709263096@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
Message-ID: <1c9264603adc240b226e.1709263098@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1709260932 -10800
#      Fri Mar 01 05:42:12 2024 +0300
# Node ID 1c9264603adc240b226e1a149c01491d62302ded
# Parent  c7c8354f99face52f3b51442360317f3e2768492
Syslog: introduced ngx_syslog_send() error logging moderation.

Errors when logging to syslog are now logged at most once per second.
This ensures that persistent errors won't flood other logs, and spontaneous
errors, such as ENOBUFS as observed on BSD systems when syslogd cannot cope
with load, or EAGAIN as seen in similar situation on Linux, won't further
overload logging subsystem, leading to more errors.

Further, errors now can only trigger reconnects at most once per second.
This ensures that persistent errors, which cannot be fixed with reconnects,
don't trigger too much unneeded work.

Additionally, in case of connection errors, such as when syslogd is not
running, connection attempts are only made once per second.

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1604,6 +1604,10 @@ ngx_connection_error(ngx_connection_t *c
     {
         switch (c->log_error) {
 
+        case NGX_ERROR_DEBUG:
+            level = NGX_LOG_DEBUG;
+            break;
+
         case NGX_ERROR_IGNORE_EMSGSIZE:
         case NGX_ERROR_IGNORE_EINVAL:
         case NGX_ERROR_IGNORE_ECONNRESET:
@@ -1615,6 +1619,9 @@ ngx_connection_error(ngx_connection_t *c
             level = NGX_LOG_ERR;
         }
 
+    } else if (c->log_error == NGX_ERROR_DEBUG) {
+        level = NGX_LOG_DEBUG;
+
     } else {
         level = NGX_LOG_ALERT;
     }
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -98,7 +98,8 @@ typedef enum {
     NGX_ERROR_INFO,
     NGX_ERROR_IGNORE_ECONNRESET,
     NGX_ERROR_IGNORE_EINVAL,
-    NGX_ERROR_IGNORE_EMSGSIZE
+    NGX_ERROR_IGNORE_EMSGSIZE,
+    NGX_ERROR_DEBUG
 } ngx_connection_log_error_e;
 
 
diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -303,6 +303,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         }
     }
 
+    if (ngx_time() == peer->error_log_time) {
+        peer->conn.log_error = NGX_ERROR_DEBUG;
+    }
+
     if (ngx_send) {
         n = ngx_send(&peer->conn, buf, len);
 
@@ -311,20 +315,27 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         n = ngx_os_io.send(&peer->conn, buf, len);
     }
 
-    if (n == NGX_ERROR) {
-
+    if (n == NGX_ERROR
+        && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         if (ngx_close_socket(peer->conn.fd) == -1) {
             ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                           ngx_close_socket_n " failed");
         }
 
         peer->conn.fd = (ngx_socket_t) -1;
+        peer->error_log_time = ngx_time();
 
-    } else if ((size_t) n != len) {
+    } else if ((size_t) n != len
+               && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
                       "send() incomplete");
+        peer->error_log_time = ngx_time();
     }
 
+    peer->conn.log_error = NGX_ERROR_ERR;
+
     return n;
 }
 
@@ -334,10 +345,15 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 {
     ngx_socket_t  fd;
 
+    if (ngx_time() == peer->connect_error_time) {
+        return NGX_ERROR;
+    }
+
     fd = ngx_socket(peer->server.sockaddr->sa_family, SOCK_DGRAM, 0);
     if (fd == (ngx_socket_t) -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_socket_n " failed");
+        peer->connect_error_time = ngx_time();
         return NGX_ERROR;
     }
 
@@ -355,6 +371,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
     peer->conn.fd = fd;
     peer->conn.log = &peer->log;
+    peer->conn.log_error = NGX_ERROR_ERR;
 
     /* UDP sockets are always ready to write */
     peer->conn.write->ready = 1;
@@ -363,6 +380,8 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
 failed:
 
+    peer->connect_error_time = ngx_time();
+
     if (ngx_close_socket(fd) == -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_close_socket_n " failed");
diff --git a/src/core/ngx_syslog.h b/src/core/ngx_syslog.h
--- a/src/core/ngx_syslog.h
+++ b/src/core/ngx_syslog.h
@@ -21,6 +21,9 @@ typedef struct {
     ngx_log_t          log;
     ngx_log_t         *logp;
 
+    time_t             error_log_time;
+    time_t             connect_error_time;
+
     unsigned           busy:1;
     unsigned           nohostname:1;
 } ngx_syslog_peer_t;



From mdounin at mdounin.ru  Fri Mar  1 03:18:19 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:19 +0300
Subject: [PATCH 3 of 5] Syslog: logging level of connect() errors
In-Reply-To: <patchbomb.1709263096@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
Message-ID: <c769217b4a5ce653bc4c.1709263099@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1709260935 -10800
#      Fri Mar 01 05:42:15 2024 +0300
# Node ID c769217b4a5ce653bc4c895e159105e375bf583f
# Parent  1c9264603adc240b226e1a149c01491d62302ded
Syslog: logging level of connect() errors.

Connecting to syslog server might legitimately fail: for example, if
syslogd is not running, and the connection is made over a unix socket.
As such, the "alert" logging level is not appropriate.  Changed
logging level to "crit" instead, similarly to the logging level used
for connect() errors in resolver.

diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -364,7 +364,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
     }
 
     if (connect(fd, peer->server.sockaddr, peer->server.socklen) == -1) {
-        ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
+        ngx_log_error(NGX_LOG_CRIT, &peer->log, ngx_socket_errno,
                       "connect() failed");
         goto failed;
     }



From mdounin at mdounin.ru  Fri Mar  1 03:18:20 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:20 +0300
Subject: [PATCH 4 of 5] Adjusted ENOBUFS logging level
In-Reply-To: <patchbomb.1709263096@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
Message-ID: <37ef53ecd6b09d37b41d.1709263100@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1709260942 -10800
#      Fri Mar 01 05:42:22 2024 +0300
# Node ID 37ef53ecd6b09d37b41d29a02e19b63e9d643994
# Parent  c769217b4a5ce653bc4c895e159105e375bf583f
Adjusted ENOBUFS logging level.

On local datagram sockets ENOBUFS might legitimately appear due to
insufficient space in socket buffers, as seen on BSD systems.
Accordingly, ngx_connection_error() now logs it with logging level
set for the particular connection errors, either "info" or "error"
(instead of "alert", which was previously used).

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1600,7 +1600,8 @@ ngx_connection_error(ngx_connection_t *c
         || err == NGX_ENETDOWN
         || err == NGX_ENETUNREACH
         || err == NGX_EHOSTDOWN
-        || err == NGX_EHOSTUNREACH)
+        || err == NGX_EHOSTUNREACH
+        || err == NGX_ENOBUFS)
     {
         switch (c->log_error) {
 
diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h
--- a/src/os/unix/ngx_errno.h
+++ b/src/os/unix/ngx_errno.h
@@ -55,6 +55,7 @@ typedef int               ngx_err_t;
 #define NGX_ELOOP         ELOOP
 #define NGX_EBADF         EBADF
 #define NGX_EMSGSIZE      EMSGSIZE
+#define NGX_ENOBUFS       ENOBUFS
 
 #if (NGX_HAVE_OPENAT)
 #define NGX_EMLINK        EMLINK
diff --git a/src/os/win32/ngx_errno.h b/src/os/win32/ngx_errno.h
--- a/src/os/win32/ngx_errno.h
+++ b/src/os/win32/ngx_errno.h
@@ -58,6 +58,7 @@ typedef DWORD                      ngx_e
 #define NGX_ELOOP                  0
 #define NGX_EBADF                  WSAEBADF
 #define NGX_EMSGSIZE               WSAEMSGSIZE
+#define NGX_ENOBUFS                WSAENOBUFS
 
 #define NGX_EALREADY               WSAEALREADY
 #define NGX_EINVAL                 WSAEINVAL



From mdounin at mdounin.ru  Fri Mar  1 03:18:21 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 01 Mar 2024 06:18:21 +0300
Subject: [PATCH 5 of 5] Adjusted logging level of ENOENT connect() errors
In-Reply-To: <patchbomb.1709263096@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
Message-ID: <c363ad6b3b23ae7fed28.1709263101@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1709260951 -10800
#      Fri Mar 01 05:42:31 2024 +0300
# Node ID c363ad6b3b23ae7fed28a7cdfa20b0cb3fc6a584
# Parent  37ef53ecd6b09d37b41d29a02e19b63e9d643994
Adjusted logging level of ENOENT connect() errors.

Connecting to unix sockets might legitimately return ENOENT if the socket
does not exists.  Changed logging level of such errors from the default
one for unknown errors, "crit", to "error".

diff --git a/src/core/ngx_resolver.c b/src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c
+++ b/src/core/ngx_resolver.c
@@ -4625,7 +4625,8 @@ ngx_tcp_connect(ngx_resolver_connection_
                 || err == NGX_ENETDOWN
                 || err == NGX_ENETUNREACH
                 || err == NGX_EHOSTDOWN
-                || err == NGX_EHOSTUNREACH)
+                || err == NGX_EHOSTUNREACH
+                || err == NGX_ENOENT)
             {
                 level = NGX_LOG_ERR;
 
diff --git a/src/event/ngx_event_connect.c b/src/event/ngx_event_connect.c
--- a/src/event/ngx_event_connect.c
+++ b/src/event/ngx_event_connect.c
@@ -231,7 +231,8 @@ ngx_event_connect_peer(ngx_peer_connecti
                 || err == NGX_ENETDOWN
                 || err == NGX_ENETUNREACH
                 || err == NGX_EHOSTDOWN
-                || err == NGX_EHOSTUNREACH)
+                || err == NGX_EHOSTUNREACH
+                || err == NGX_ENOENT)
             {
                 level = NGX_LOG_ERR;
 



From vl at inspert.ru  Thu Mar  7 14:24:47 2024
From: vl at inspert.ru (Vladimir Homutov)
Date: Thu, 7 Mar 2024 17:24:47 +0300
Subject: [PATCH 2 of 5] Syslog: introduced ngx_syslog_send() error
 logging moderation
In-Reply-To: <1c9264603adc240b226e.1709263098@vm-bsd.mdounin.ru>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
 <1c9264603adc240b226e.1709263098@vm-bsd.mdounin.ru>
Message-ID: <ZenOL-I2x8o0IG8q@vlws>

On Fri, Mar 01, 2024 at 06:18:18AM +0300, Maxim Dounin wrote:
> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1709260932 -10800
> #      Fri Mar 01 05:42:12 2024 +0300
> # Node ID 1c9264603adc240b226e1a149c01491d62302ded
> # Parent  c7c8354f99face52f3b51442360317f3e2768492
> Syslog: introduced ngx_syslog_send() error logging moderation.
>
> Errors when logging to syslog are now logged at most once per second.
> This ensures that persistent errors won't flood other logs, and spontaneous
> errors, such as ENOBUFS as observed on BSD systems when syslogd cannot cope
> with load, or EAGAIN as seen in similar situation on Linux, won't further
> overload logging subsystem, leading to more errors.
>
> Further, errors now can only trigger reconnects at most once per second.
> This ensures that persistent errors, which cannot be fixed with reconnects,
> don't trigger too much unneeded work.
>
> Additionally, in case of connection errors, such as when syslogd is not
> running, connection attempts are only made once per second.

The change is good, but I can't get the remaining logic:
We don't do reconnects too often (good), but we are still trying
to send into socket that responded with error last time.

Previously, we closed such socket (the goal was to recover from
persistent error).

With the patch we are trying to recoverr once per second, but still
trying to use the (presumably) bad socket.
Maybe we should instead just stop sending anything for the second?

If it seems too much (we can produce a lot of logs during a second and
if an error was random, things would run just fine), then I guess that
we can introduce a counter: close the socket in case of N successive
errors, and put it into moderation mode, and retry again in 1 second.

Other patches look good for me.


>
> diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
> --- a/src/core/ngx_connection.c
> +++ b/src/core/ngx_connection.c
> @@ -1604,6 +1604,10 @@ ngx_connection_error(ngx_connection_t *c
>      {
>          switch (c->log_error) {
>
> +        case NGX_ERROR_DEBUG:
> +            level = NGX_LOG_DEBUG;
> +            break;
> +
>          case NGX_ERROR_IGNORE_EMSGSIZE:
>          case NGX_ERROR_IGNORE_EINVAL:
>          case NGX_ERROR_IGNORE_ECONNRESET:
> @@ -1615,6 +1619,9 @@ ngx_connection_error(ngx_connection_t *c
>              level = NGX_LOG_ERR;
>          }
>
> +    } else if (c->log_error == NGX_ERROR_DEBUG) {
> +        level = NGX_LOG_DEBUG;
> +
>      } else {
>          level = NGX_LOG_ALERT;
>      }
> diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
> --- a/src/core/ngx_connection.h
> +++ b/src/core/ngx_connection.h
> @@ -98,7 +98,8 @@ typedef enum {
>      NGX_ERROR_INFO,
>      NGX_ERROR_IGNORE_ECONNRESET,
>      NGX_ERROR_IGNORE_EINVAL,
> -    NGX_ERROR_IGNORE_EMSGSIZE
> +    NGX_ERROR_IGNORE_EMSGSIZE,
> +    NGX_ERROR_DEBUG
>  } ngx_connection_log_error_e;
>
>
> diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
> --- a/src/core/ngx_syslog.c
> +++ b/src/core/ngx_syslog.c
> @@ -303,6 +303,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
>          }
>      }
>
> +    if (ngx_time() == peer->error_log_time) {
> +        peer->conn.log_error = NGX_ERROR_DEBUG;
> +    }
> +
>      if (ngx_send) {
>          n = ngx_send(&peer->conn, buf, len);
>
> @@ -311,20 +315,27 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
>          n = ngx_os_io.send(&peer->conn, buf, len);
>      }
>
> -    if (n == NGX_ERROR) {
> -
> +    if (n == NGX_ERROR
> +        && peer->conn.log_error != NGX_ERROR_DEBUG)
> +    {
>          if (ngx_close_socket(peer->conn.fd) == -1) {
>              ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
>                            ngx_close_socket_n " failed");
>          }
>
>          peer->conn.fd = (ngx_socket_t) -1;
> +        peer->error_log_time = ngx_time();
>
> -    } else if ((size_t) n != len) {
> +    } else if ((size_t) n != len
> +               && peer->conn.log_error != NGX_ERROR_DEBUG)
> +    {
>          ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
>                        "send() incomplete");
> +        peer->error_log_time = ngx_time();
>      }
>
> +    peer->conn.log_error = NGX_ERROR_ERR;
> +
>      return n;
>  }
>
> @@ -334,10 +345,15 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
>  {
>      ngx_socket_t  fd;
>
> +    if (ngx_time() == peer->connect_error_time) {
> +        return NGX_ERROR;
> +    }
> +
>      fd = ngx_socket(peer->server.sockaddr->sa_family, SOCK_DGRAM, 0);
>      if (fd == (ngx_socket_t) -1) {
>          ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
>                        ngx_socket_n " failed");
> +        peer->connect_error_time = ngx_time();
>          return NGX_ERROR;
>      }
>
> @@ -355,6 +371,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
>
>      peer->conn.fd = fd;
>      peer->conn.log = &peer->log;
> +    peer->conn.log_error = NGX_ERROR_ERR;
>
>      /* UDP sockets are always ready to write */
>      peer->conn.write->ready = 1;
> @@ -363,6 +380,8 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
>
>  failed:
>
> +    peer->connect_error_time = ngx_time();
> +
>      if (ngx_close_socket(fd) == -1) {
>          ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
>                        ngx_close_socket_n " failed");
> diff --git a/src/core/ngx_syslog.h b/src/core/ngx_syslog.h
> --- a/src/core/ngx_syslog.h
> +++ b/src/core/ngx_syslog.h
> @@ -21,6 +21,9 @@ typedef struct {
>      ngx_log_t          log;
>      ngx_log_t         *logp;
>
> +    time_t             error_log_time;
> +    time_t             connect_error_time;
> +
>      unsigned           busy:1;
>      unsigned           nohostname:1;
>  } ngx_syslog_peer_t;
>
> --
> nginx-devel mailing list
> nginx-devel at freenginx.org
> https://freenginx.org/mailman/listinfo/nginx-devel


From mdounin at mdounin.ru  Thu Mar  7 16:06:03 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 7 Mar 2024 19:06:03 +0300
Subject: [PATCH 2 of 5] Syslog: introduced ngx_syslog_send() error
 logging moderation
In-Reply-To: <ZenOL-I2x8o0IG8q@vlws>
References: <patchbomb.1709263096@vm-bsd.mdounin.ru>
 <1c9264603adc240b226e.1709263098@vm-bsd.mdounin.ru>
 <ZenOL-I2x8o0IG8q@vlws>
Message-ID: <Zenl66IpEynYOUZJ@mdounin.ru>

Hello!

On Thu, Mar 07, 2024 at 05:24:47PM +0300, Vladimir Homutov wrote:

> On Fri, Mar 01, 2024 at 06:18:18AM +0300, Maxim Dounin wrote:
> > # HG changeset patch
> > # User Maxim Dounin <mdounin at mdounin.ru>
> > # Date 1709260932 -10800
> > #      Fri Mar 01 05:42:12 2024 +0300
> > # Node ID 1c9264603adc240b226e1a149c01491d62302ded
> > # Parent  c7c8354f99face52f3b51442360317f3e2768492
> > Syslog: introduced ngx_syslog_send() error logging moderation.
> >
> > Errors when logging to syslog are now logged at most once per second.
> > This ensures that persistent errors won't flood other logs, and spontaneous
> > errors, such as ENOBUFS as observed on BSD systems when syslogd cannot cope
> > with load, or EAGAIN as seen in similar situation on Linux, won't further
> > overload logging subsystem, leading to more errors.
> >
> > Further, errors now can only trigger reconnects at most once per second.
> > This ensures that persistent errors, which cannot be fixed with reconnects,
> > don't trigger too much unneeded work.
> >
> > Additionally, in case of connection errors, such as when syslogd is not
> > running, connection attempts are only made once per second.
> 
> The change is good, but I can't get the remaining logic:
> We don't do reconnects too often (good), but we are still trying
> to send into socket that responded with error last time.
> 
> Previously, we closed such socket (the goal was to recover from
> persistent error).
> 
> With the patch we are trying to recoverr once per second, but still
> trying to use the (presumably) bad socket.
> Maybe we should instead just stop sending anything for the second?

At least some errors on datagram sockets, such as ENOBUFS on BSD 
systems, are transient, and basically mean that the particular 
datagram was lost, while other datagrams sent to the same socket 
might sill succeed.  Reconnecting on each lost datagram is useless 
(and will require additional resources), but sending other 
datagrams to the same socket might be still beneficial - they 
might be properly delivered. 

The idea is that if we've encountered an error which wasn't fixed 
by a reconnect, we are probably dealing with one of such errors, 
and hence we suppress reconnects for a while.

Suppressing logging at all does not seem to be needed, since 
sending which results in errors, as long as it doesn't trigger 
additional error logging, takes roughly the same resources as 
normal sending, and hence can be used safely.  At least I'm not 
aware of anything similar to ENOSPC errors, which are known to 
require lots of resources in some cases (and therefore we suppress 
file logging for a second on ENOSPC).

> If it seems too much (we can produce a lot of logs during a second and
> if an error was random, things would run just fine), then I guess that
> we can introduce a counter: close the socket in case of N successive
> errors, and put it into moderation mode, and retry again in 1 second.
> 
> Other patches look good for me.

Thanks for looking.

[...]

-- 
Maxim Dounin
http://mdounin.ru/


From mdounin at mdounin.ru  Wed Mar 13 15:52:32 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Wed, 13 Mar 2024 18:52:32 +0300
Subject: [nginx] Syslog: fixed duplicate errors with access logging to sy...
Message-ID: <hg.c7c8354f99fa.1710345152.-4005316334648894766@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/c7c8354f99fa
branches:  
changeset: 9224:c7c8354f99fa
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:09 2024 +0300
description:
Syslog: fixed duplicate errors with access logging to syslog.

The ngx_syslog_send() function logs errors itself, so there is no need
to additionally log errors in the caller, notably access log modules in
http and stream.

To ensure that incomplete and blocked writes are also logged, appropriate
logging added to ngx_syslog_send().

diffstat:

 src/core/ngx_syslog.c                  |   4 ++++
 src/http/modules/ngx_http_log_module.c |  17 ++---------------
 src/stream/ngx_stream_log_module.c     |  17 ++---------------
 3 files changed, 8 insertions(+), 30 deletions(-)

diffs (82 lines):

diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -319,6 +319,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         }
 
         peer->conn.fd = (ngx_socket_t) -1;
+
+    } else if ((size_t) n != len) {
+        ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
+                      "send() incomplete");
     }
 
     return n;
diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c
--- a/src/http/modules/ngx_http_log_module.c
+++ b/src/http/modules/ngx_http_log_module.c
@@ -254,8 +254,7 @@ static ngx_int_t
 ngx_http_log_handler(ngx_http_request_t *r)
 {
     u_char                   *line, *p;
-    size_t                    len, size;
-    ssize_t                   n;
+    size_t                    len;
     ngx_str_t                 val;
     ngx_uint_t                i, l;
     ngx_http_log_t           *log;
@@ -376,19 +375,7 @@ ngx_http_log_handler(ngx_http_request_t 
 
         if (log[l].syslog_peer) {
 
-            size = p - line;
-
-            n = ngx_syslog_send(log[l].syslog_peer, line, size);
-
-            if (n < 0) {
-                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                              "send() to syslog failed");
-
-            } else if ((size_t) n != size) {
-                ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                              "send() to syslog has written only %z of %uz",
-                              n, size);
-            }
+            (void) ngx_syslog_send(log[l].syslog_peer, line, p - line);
 
             continue;
         }
diff --git a/src/stream/ngx_stream_log_module.c b/src/stream/ngx_stream_log_module.c
--- a/src/stream/ngx_stream_log_module.c
+++ b/src/stream/ngx_stream_log_module.c
@@ -201,8 +201,7 @@ static ngx_int_t
 ngx_stream_log_handler(ngx_stream_session_t *s)
 {
     u_char                     *line, *p;
-    size_t                      len, size;
-    ssize_t                     n;
+    size_t                      len;
     ngx_str_t                   val;
     ngx_uint_t                  i, l;
     ngx_stream_log_t           *log;
@@ -324,19 +323,7 @@ ngx_stream_log_handler(ngx_stream_sessio
 
         if (log[l].syslog_peer) {
 
-            size = p - line;
-
-            n = ngx_syslog_send(log[l].syslog_peer, line, size);
-
-            if (n < 0) {
-                ngx_log_error(NGX_LOG_WARN, s->connection->log, 0,
-                              "send() to syslog failed");
-
-            } else if ((size_t) n != size) {
-                ngx_log_error(NGX_LOG_WARN, s->connection->log, 0,
-                              "send() to syslog has written only %z of %uz",
-                              n, size);
-            }
+            (void) ngx_syslog_send(log[l].syslog_peer, line, p - line);
 
             continue;
         }


From mdounin at mdounin.ru  Wed Mar 13 15:52:32 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Wed, 13 Mar 2024 18:52:32 +0300
Subject: [nginx] Syslog: introduced ngx_syslog_send() error logging moder...
Message-ID: <hg.1c9264603adc.1710345152.-4005316334648894766@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/1c9264603adc
branches:  
changeset: 9225:1c9264603adc
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:12 2024 +0300
description:
Syslog: introduced ngx_syslog_send() error logging moderation.

Errors when logging to syslog are now logged at most once per second.
This ensures that persistent errors won't flood other logs, and spontaneous
errors, such as ENOBUFS as observed on BSD systems when syslogd cannot cope
with load, or EAGAIN as seen in similar situation on Linux, won't further
overload logging subsystem, leading to more errors.

Further, errors now can only trigger reconnects at most once per second.
This ensures that persistent errors, which cannot be fixed with reconnects,
don't trigger too much unneeded work.

Additionally, in case of connection errors, such as when syslogd is not
running, connection attempts are only made once per second.

diffstat:

 src/core/ngx_connection.c |   7 +++++++
 src/core/ngx_connection.h |   3 ++-
 src/core/ngx_syslog.c     |  25 ++++++++++++++++++++++---
 src/core/ngx_syslog.h     |   3 +++
 4 files changed, 34 insertions(+), 4 deletions(-)

diffs (128 lines):

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1604,6 +1604,10 @@ ngx_connection_error(ngx_connection_t *c
     {
         switch (c->log_error) {
 
+        case NGX_ERROR_DEBUG:
+            level = NGX_LOG_DEBUG;
+            break;
+
         case NGX_ERROR_IGNORE_EMSGSIZE:
         case NGX_ERROR_IGNORE_EINVAL:
         case NGX_ERROR_IGNORE_ECONNRESET:
@@ -1615,6 +1619,9 @@ ngx_connection_error(ngx_connection_t *c
             level = NGX_LOG_ERR;
         }
 
+    } else if (c->log_error == NGX_ERROR_DEBUG) {
+        level = NGX_LOG_DEBUG;
+
     } else {
         level = NGX_LOG_ALERT;
     }
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -98,7 +98,8 @@ typedef enum {
     NGX_ERROR_INFO,
     NGX_ERROR_IGNORE_ECONNRESET,
     NGX_ERROR_IGNORE_EINVAL,
-    NGX_ERROR_IGNORE_EMSGSIZE
+    NGX_ERROR_IGNORE_EMSGSIZE,
+    NGX_ERROR_DEBUG
 } ngx_connection_log_error_e;
 
 
diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -303,6 +303,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         }
     }
 
+    if (ngx_time() == peer->error_log_time) {
+        peer->conn.log_error = NGX_ERROR_DEBUG;
+    }
+
     if (ngx_send) {
         n = ngx_send(&peer->conn, buf, len);
 
@@ -311,20 +315,27 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         n = ngx_os_io.send(&peer->conn, buf, len);
     }
 
-    if (n == NGX_ERROR) {
-
+    if (n == NGX_ERROR
+        && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         if (ngx_close_socket(peer->conn.fd) == -1) {
             ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                           ngx_close_socket_n " failed");
         }
 
         peer->conn.fd = (ngx_socket_t) -1;
+        peer->error_log_time = ngx_time();
 
-    } else if ((size_t) n != len) {
+    } else if ((size_t) n != len
+               && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
                       "send() incomplete");
+        peer->error_log_time = ngx_time();
     }
 
+    peer->conn.log_error = NGX_ERROR_ERR;
+
     return n;
 }
 
@@ -334,10 +345,15 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 {
     ngx_socket_t  fd;
 
+    if (ngx_time() == peer->connect_error_time) {
+        return NGX_ERROR;
+    }
+
     fd = ngx_socket(peer->server.sockaddr->sa_family, SOCK_DGRAM, 0);
     if (fd == (ngx_socket_t) -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_socket_n " failed");
+        peer->connect_error_time = ngx_time();
         return NGX_ERROR;
     }
 
@@ -355,6 +371,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
     peer->conn.fd = fd;
     peer->conn.log = &peer->log;
+    peer->conn.log_error = NGX_ERROR_ERR;
 
     /* UDP sockets are always ready to write */
     peer->conn.write->ready = 1;
@@ -363,6 +380,8 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
 failed:
 
+    peer->connect_error_time = ngx_time();
+
     if (ngx_close_socket(fd) == -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_close_socket_n " failed");
diff --git a/src/core/ngx_syslog.h b/src/core/ngx_syslog.h
--- a/src/core/ngx_syslog.h
+++ b/src/core/ngx_syslog.h
@@ -21,6 +21,9 @@ typedef struct {
     ngx_log_t          log;
     ngx_log_t         *logp;
 
+    time_t             error_log_time;
+    time_t             connect_error_time;
+
     unsigned           busy:1;
     unsigned           nohostname:1;
 } ngx_syslog_peer_t;


From mdounin at mdounin.ru  Wed Mar 13 15:52:32 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Wed, 13 Mar 2024 18:52:32 +0300
Subject: [nginx] Syslog: logging level of connect() errors.
Message-ID: <hg.c769217b4a5c.1710345152.-4005316334648894766@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/c769217b4a5c
branches:  
changeset: 9226:c769217b4a5c
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:15 2024 +0300
description:
Syslog: logging level of connect() errors.

Connecting to syslog server might legitimately fail: for example, if
syslogd is not running, and the connection is made over a unix socket.
As such, the "alert" logging level is not appropriate.  Changed
logging level to "crit" instead, similarly to the logging level used
for connect() errors in resolver.

diffstat:

 src/core/ngx_syslog.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -364,7 +364,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
     }
 
     if (connect(fd, peer->server.sockaddr, peer->server.socklen) == -1) {
-        ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
+        ngx_log_error(NGX_LOG_CRIT, &peer->log, ngx_socket_errno,
                       "connect() failed");
         goto failed;
     }


From mdounin at mdounin.ru  Wed Mar 13 15:52:32 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Wed, 13 Mar 2024 18:52:32 +0300
Subject: [nginx] Adjusted ENOBUFS logging level.
Message-ID: <hg.37ef53ecd6b0.1710345152.-4005316334648894766@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/37ef53ecd6b0
branches:  
changeset: 9227:37ef53ecd6b0
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:22 2024 +0300
description:
Adjusted ENOBUFS logging level.

On local datagram sockets ENOBUFS might legitimately appear due to
insufficient space in socket buffers, as seen on BSD systems.
Accordingly, ngx_connection_error() now logs it with logging level
set for the particular connection errors, either "info" or "error"
(instead of "alert", which was previously used).

diffstat:

 src/core/ngx_connection.c |  3 ++-
 src/os/unix/ngx_errno.h   |  1 +
 src/os/win32/ngx_errno.h  |  1 +
 3 files changed, 4 insertions(+), 1 deletions(-)

diffs (35 lines):

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1600,7 +1600,8 @@ ngx_connection_error(ngx_connection_t *c
         || err == NGX_ENETDOWN
         || err == NGX_ENETUNREACH
         || err == NGX_EHOSTDOWN
-        || err == NGX_EHOSTUNREACH)
+        || err == NGX_EHOSTUNREACH
+        || err == NGX_ENOBUFS)
     {
         switch (c->log_error) {
 
diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h
--- a/src/os/unix/ngx_errno.h
+++ b/src/os/unix/ngx_errno.h
@@ -55,6 +55,7 @@ typedef int               ngx_err_t;
 #define NGX_ELOOP         ELOOP
 #define NGX_EBADF         EBADF
 #define NGX_EMSGSIZE      EMSGSIZE
+#define NGX_ENOBUFS       ENOBUFS
 
 #if (NGX_HAVE_OPENAT)
 #define NGX_EMLINK        EMLINK
diff --git a/src/os/win32/ngx_errno.h b/src/os/win32/ngx_errno.h
--- a/src/os/win32/ngx_errno.h
+++ b/src/os/win32/ngx_errno.h
@@ -58,6 +58,7 @@ typedef DWORD                      ngx_e
 #define NGX_ELOOP                  0
 #define NGX_EBADF                  WSAEBADF
 #define NGX_EMSGSIZE               WSAEMSGSIZE
+#define NGX_ENOBUFS                WSAENOBUFS
 
 #define NGX_EALREADY               WSAEALREADY
 #define NGX_EINVAL                 WSAEINVAL


From mdounin at mdounin.ru  Wed Mar 13 15:52:32 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Wed, 13 Mar 2024 18:52:32 +0300
Subject: [nginx] Adjusted logging level of ENOENT connect() errors.
Message-ID: <hg.c363ad6b3b23.1710345152.-4005316334648894766@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/c363ad6b3b23
branches:  
changeset: 9228:c363ad6b3b23
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:31 2024 +0300
description:
Adjusted logging level of ENOENT connect() errors.

Connecting to unix sockets might legitimately return ENOENT if the socket
does not exists.  Changed logging level of such errors from the default
one for unknown errors, "crit", to "error".

diffstat:

 src/core/ngx_resolver.c       |  3 ++-
 src/event/ngx_event_connect.c |  3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diffs (26 lines):

diff --git a/src/core/ngx_resolver.c b/src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c
+++ b/src/core/ngx_resolver.c
@@ -4625,7 +4625,8 @@ ngx_tcp_connect(ngx_resolver_connection_
                 || err == NGX_ENETDOWN
                 || err == NGX_ENETUNREACH
                 || err == NGX_EHOSTDOWN
-                || err == NGX_EHOSTUNREACH)
+                || err == NGX_EHOSTUNREACH
+                || err == NGX_ENOENT)
             {
                 level = NGX_LOG_ERR;
 
diff --git a/src/event/ngx_event_connect.c b/src/event/ngx_event_connect.c
--- a/src/event/ngx_event_connect.c
+++ b/src/event/ngx_event_connect.c
@@ -231,7 +231,8 @@ ngx_event_connect_peer(ngx_peer_connecti
                 || err == NGX_ENETDOWN
                 || err == NGX_ENETUNREACH
                 || err == NGX_EHOSTDOWN
-                || err == NGX_EHOSTUNREACH)
+                || err == NGX_EHOSTUNREACH
+                || err == NGX_ENOENT)
             {
                 level = NGX_LOG_ERR;
 


From mdounin at mdounin.ru  Thu Mar 14 00:17:49 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Thu, 14 Mar 2024 03:17:49 +0300
Subject: [nginx-tests] Tests: fixed h3_limit_req.t spurious failures.
Message-ID: <hg.1867428f1673.1710375469.-8200033269367280750@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx-tests/rev/1867428f1673
branches:  
changeset: 1951:1867428f1673
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Mar 14 02:25:49 2024 +0300
description:
Tests: fixed h3_limit_req.t spurious failures.

In the "reset stream - cancellation" test, HTTP/3 stream is closed without
sending the request body when the request is waiting in the limit_req
module, and this results in error 444.  However, when the request is received
with some minor delay due to system load, it is not delayed by limit_req,
and the stream is closed during reading the request body, which results
in error 400 instead, breaking the test.

Fix is to introduce yet another request before the "reset stream" test,
so the stream in question is always delayed by limit_req.

diffstat:

 h3_limit_req.t |  10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)

diffs (27 lines):

diff --git a/h3_limit_req.t b/h3_limit_req.t
--- a/h3_limit_req.t
+++ b/h3_limit_req.t
@@ -24,7 +24,7 @@ select STDERR; $| = 1;
 select STDOUT; $| = 1;
 
 my $t = Test::Nginx->new()->has(qw/http http_v3 proxy limit_req cryptx/)
-	->has_daemon('openssl')->plan(6);
+	->has_daemon('openssl')->plan(7);
 
 $t->write_file_expand('nginx.conf', <<'EOF');
 
@@ -131,6 +131,14 @@ select undef, undef, undef, 1.1;
 ($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
 is($frame->{headers}->{':status'}, 200, 'request body - limit req - empty');
 
+# another request
+
+$sid = $s->new_stream({ path => '/' });
+$frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
+
+($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
+is($frame->{headers}->{':status'}, '200', 'request body - limit req - next');
+
 # detect RESET_STREAM while request is delayed
 
 $s = Test::Nginx::HTTP3->new();


From lafiel at elven.pw  Fri Mar 15 11:01:13 2024
From: lafiel at elven.pw (Lafiel)
Date: Fri, 15 Mar 2024 14:01:13 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
Message-ID: <21cfa15983117d64856b62197d96d3aa@elven.pw>

Hello, 
Maksim

I 
fixed 
a 
few 
commits.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mime_types_01.patch
Type: text/x-diff
Size: 8551 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240315/a90d567f/attachment.patch>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240315/a90d567f/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240315/a90d567f/attachment.sig>

From mdounin at mdounin.ru  Fri Mar 15 18:14:14 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:14 +0300
Subject: [PATCH 00 of 10] read hardening
Message-ID: <patchbomb.1710526454@vm-bsd.mdounin.ru>

Hello!

Here is a patch series which introduces various hardening in data
reading code paths.  Review and testing appreciated.

-- 
Maxim Dounin



From mdounin at mdounin.ru  Fri Mar 15 18:14:15 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:15 +0300
Subject: [PATCH 01 of 10] Mail: switched to posted events when resuming reading
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <dc9a4c5573411f8f6115.1710526455@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526239 -10800
#      Fri Mar 15 21:10:39 2024 +0300
# Node ID dc9a4c5573411f8f6115db56d201ce0fa2c002fc
# Parent  c363ad6b3b23ae7fed28a7cdfa20b0cb3fc6a584
Mail: switched to posted events when resuming reading.

When resuming reading after ngx_mail_send(), switched to using posted
events instead of a direct event handler call.  This ensures limited
stack usage when processing multiple pipelined commands.

diff --git a/src/mail/ngx_mail_handler.c b/src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c
+++ b/src/mail/ngx_mail_handler.c
@@ -799,7 +799,7 @@ ngx_mail_send(ngx_event_t *wev)
         }
 
         if (s->blocked) {
-            c->read->handler(c->read);
+            ngx_post_event(c->read, &ngx_posted_events);
         }
 
         return;



From mdounin at mdounin.ru  Fri Mar 15 18:14:16 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:16 +0300
Subject: [PATCH 02 of 10] Mail: handling of read buffer filled with commands
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <1d26e6050416a6c8a86d.1710526456@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526241 -10800
#      Fri Mar 15 21:10:41 2024 +0300
# Node ID 1d26e6050416a6c8a86d83119d894f7f9f4f5a94
# Parent  dc9a4c5573411f8f6115db56d201ce0fa2c002fc
Mail: handling of read buffer filled with commands.

If the whole read buffer was filled with commands during authentication,
exactly matching the buffer boundary, this did not cause the "client sent
too long command" error, but might result in read events being disabled
for the connection, notably with level-triggered event methods, leading
to a connection hang.

Fix is to ensure that s->blocked is set in case of c->read->ready, so
command reading is resumed after sending responses to previous commands.

diff --git a/src/mail/ngx_mail_imap_handler.c b/src/mail/ngx_mail_imap_handler.c
--- a/src/mail/ngx_mail_imap_handler.c
+++ b/src/mail/ngx_mail_imap_handler.c
@@ -226,7 +226,7 @@ ngx_mail_imap_auth_state(ngx_event_t *re
         ngx_str_set(&s->out, imap_next);
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 
diff --git a/src/mail/ngx_mail_pop3_handler.c b/src/mail/ngx_mail_pop3_handler.c
--- a/src/mail/ngx_mail_pop3_handler.c
+++ b/src/mail/ngx_mail_pop3_handler.c
@@ -262,7 +262,7 @@ ngx_mail_pop3_auth_state(ngx_event_t *re
         }
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 
diff --git a/src/mail/ngx_mail_smtp_handler.c b/src/mail/ngx_mail_smtp_handler.c
--- a/src/mail/ngx_mail_smtp_handler.c
+++ b/src/mail/ngx_mail_smtp_handler.c
@@ -550,7 +550,7 @@ ngx_mail_smtp_auth_state(ngx_event_t *re
         }
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 



From mdounin at mdounin.ru  Fri Mar 15 18:14:17 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:17 +0300
Subject: [PATCH 03 of 10] Mail: handling of pipelined commands which cross
 buffer boundary
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <970059108845286afec7.1710526457@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526243 -10800
#      Fri Mar 15 21:10:43 2024 +0300
# Node ID 970059108845286afec7ae37ba5f55b0cf30feef
# Parent  1d26e6050416a6c8a86d83119d894f7f9f4f5a94
Mail: handling of pipelined commands which cross buffer boundary.

Previously, pipelined commands which cross buffer boundary were rejected
as too long, even if the command itself was short enough to be handled
within the client buffer.

Fix is to move non-processed commands to the start of the buffer after
the previous command is fully processed.

diff --git a/src/mail/ngx_mail_imap_handler.c b/src/mail/ngx_mail_imap_handler.c
--- a/src/mail/ngx_mail_imap_handler.c
+++ b/src/mail/ngx_mail_imap_handler.c
@@ -102,6 +102,7 @@ void
 ngx_mail_imap_auth_state(ngx_event_t *rev)
 {
     u_char              *p;
+    size_t               n;
     ngx_int_t            rc;
     ngx_uint_t           tag;
     ngx_connection_t    *c;
@@ -285,6 +286,12 @@ ngx_mail_imap_auth_state(ngx_event_t *re
             if (s->buffer->pos == s->buffer->last) {
                 s->buffer->pos = s->buffer->start;
                 s->buffer->last = s->buffer->start;
+
+            } else {
+                n = s->buffer->last - s->buffer->pos;
+                ngx_memmove(s->buffer->start, s->buffer->pos, n);
+                s->buffer->pos = s->buffer->start;
+                s->buffer->last = s->buffer->start + n;
             }
 
             s->tag.len = 0;
diff --git a/src/mail/ngx_mail_pop3_handler.c b/src/mail/ngx_mail_pop3_handler.c
--- a/src/mail/ngx_mail_pop3_handler.c
+++ b/src/mail/ngx_mail_pop3_handler.c
@@ -119,6 +119,7 @@ ngx_mail_pop3_init_protocol(ngx_event_t 
 void
 ngx_mail_pop3_auth_state(ngx_event_t *rev)
 {
+    size_t               n;
     ngx_int_t            rc;
     ngx_connection_t    *c;
     ngx_mail_session_t  *s;
@@ -291,6 +292,12 @@ ngx_mail_pop3_auth_state(ngx_event_t *re
         if (s->buffer->pos == s->buffer->last) {
             s->buffer->pos = s->buffer->start;
             s->buffer->last = s->buffer->start;
+
+        } else {
+            n = s->buffer->last - s->buffer->pos;
+            ngx_memmove(s->buffer->start, s->buffer->pos, n);
+            s->buffer->pos = s->buffer->start;
+            s->buffer->last = s->buffer->start + n;
         }
 
         if (s->state) {
diff --git a/src/mail/ngx_mail_smtp_handler.c b/src/mail/ngx_mail_smtp_handler.c
--- a/src/mail/ngx_mail_smtp_handler.c
+++ b/src/mail/ngx_mail_smtp_handler.c
@@ -430,6 +430,7 @@ ngx_mail_smtp_create_buffer(ngx_mail_ses
 void
 ngx_mail_smtp_auth_state(ngx_event_t *rev)
 {
+    size_t               n;
     ngx_int_t            rc;
     ngx_connection_t    *c;
     ngx_mail_session_t  *s;
@@ -577,6 +578,12 @@ ngx_mail_smtp_auth_state(ngx_event_t *re
         if (s->buffer->pos == s->buffer->last) {
             s->buffer->pos = s->buffer->start;
             s->buffer->last = s->buffer->start;
+
+        } else {
+            n = s->buffer->last - s->buffer->pos;
+            ngx_memmove(s->buffer->start, s->buffer->pos, n);
+            s->buffer->pos = s->buffer->start;
+            s->buffer->last = s->buffer->start + n;
         }
 
         if (s->state) {



From mdounin at mdounin.ru  Fri Mar 15 18:14:18 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:18 +0300
Subject: [PATCH 04 of 10] Mail: max_commands directive
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <c2e67d39c6daab0c67b6.1710526458@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526245 -10800
#      Fri Mar 15 21:10:45 2024 +0300
# Node ID c2e67d39c6daab0c67b68aaa199f42975c37d676
# Parent  970059108845286afec7ae37ba5f55b0cf30feef
Mail: max_commands directive.

The directive specifies the maximum number of commands allowed during
authentication, after which the connection is closed.  The default
limit is 1000, which is not expected to affect any well-behaving clients,
since authentication usually requires at most several commands, though
will effectively stop malicious clients from flooding the server with
with commands.

diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h
+++ b/src/mail/ngx_mail.h
@@ -116,6 +116,7 @@ typedef struct {
     ngx_msec_t              resolver_timeout;
 
     ngx_uint_t              max_errors;
+    ngx_uint_t              max_commands;
 
     ngx_str_t               server_name;
 
@@ -234,6 +235,7 @@ typedef struct {
     ngx_array_t             args;
 
     ngx_uint_t              errors;
+    ngx_uint_t              commands;
     ngx_uint_t              login_attempt;
 
     /* used to parse POP3/IMAP/SMTP command */
diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c
+++ b/src/mail/ngx_mail_core_module.c
@@ -92,6 +92,13 @@ static ngx_command_t  ngx_mail_core_comm
       offsetof(ngx_mail_core_srv_conf_t, max_errors),
       NULL },
 
+    { ngx_string("max_commands"),
+      NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_MAIL_SRV_CONF_OFFSET,
+      offsetof(ngx_mail_core_srv_conf_t, max_commands),
+      NULL },
+
       ngx_null_command
 };
 
@@ -171,6 +178,7 @@ ngx_mail_core_create_srv_conf(ngx_conf_t
     cscf->resolver_timeout = NGX_CONF_UNSET_MSEC;
 
     cscf->max_errors = NGX_CONF_UNSET_UINT;
+    cscf->max_commands = NGX_CONF_UNSET_UINT;
 
     cscf->resolver = NGX_CONF_UNSET_PTR;
 
@@ -192,6 +200,7 @@ ngx_mail_core_merge_srv_conf(ngx_conf_t 
                               30000);
 
     ngx_conf_merge_uint_value(conf->max_errors, prev->max_errors, 5);
+    ngx_conf_merge_uint_value(conf->max_commands, prev->max_commands, 1000);
 
     ngx_conf_merge_str_value(conf->server_name, prev->server_name, "");
 
diff --git a/src/mail/ngx_mail_handler.c b/src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c
+++ b/src/mail/ngx_mail_handler.c
@@ -896,6 +896,18 @@ ngx_mail_read_command(ngx_mail_session_t
         return NGX_ERROR;
     }
 
+    s->commands++;
+
+    if (s->commands > cscf->max_commands) {
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "client sent too many commands");
+
+        s->quit = 1;
+
+        return NGX_MAIL_PARSE_INVALID_COMMAND;
+    }
+
     return NGX_OK;
 }
 



From mdounin at mdounin.ru  Fri Mar 15 18:14:19 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:19 +0300
Subject: [PATCH 05 of 10] Upstream: improved c->read->ready flag handling
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <74525610a3b6d8d69f7e.1710526459@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526248 -10800
#      Fri Mar 15 21:10:48 2024 +0300
# Node ID 74525610a3b6d8d69f7e0082552494a889aa858e
# Parent  c2e67d39c6daab0c67b68aaa199f42975c37d676
Upstream: improved c->read->ready flag handling.

Previously, reading the upstream response headers did not check the
c->read->ready flag.  Now the flag is checked.

This ensures that ev->available checks as introduced in 7583:efd71d49bde0
and 7584:9d2ad2fb4423 will be able to prevent reading from the socket
when using event methods other than kqueue or epoll, and when using SSL.

This might be important to avoid looping for a long time when working
with fast upstream servers over protocols where large chunks of data can
be skipped while reading response headers, notably FastCGI and gRPC.

diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -2437,7 +2437,12 @@ ngx_http_upstream_process_header(ngx_htt
 
     for ( ;; ) {
 
-        n = c->recv(c, u->buffer.last, u->buffer.end - u->buffer.last);
+        if (c->read->ready) {
+            n = c->recv(c, u->buffer.last, u->buffer.end - u->buffer.last);
+
+        } else {
+            n = NGX_AGAIN;
+        }
 
         if (n == NGX_AGAIN) {
 #if 0



From mdounin at mdounin.ru  Fri Mar 15 18:14:20 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:20 +0300
Subject: [PATCH 06 of 10] Request body: explicit handling of NGX_AGAIN
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <3cd5f5e21b771ba54709.1710526460@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526250 -10800
#      Fri Mar 15 21:10:50 2024 +0300
# Node ID 3cd5f5e21b771ba547097e729642cf568ac49932
# Parent  74525610a3b6d8d69f7e0082552494a889aa858e
Request body: explicit handling of NGX_AGAIN.

Request body reading indirectly uses the "do { c->recv() } while
(c->read->ready)" form, which is not really correct, as for example
with SSL c->read->ready may be still set when c->recv() returns NGX_AGAIN
due to SSL_ERROR_WANT_WRITE (see 7351:2b5528023f6b), and therefore this
form might be an infinite loop.

Added explicit NGX_AGAIN handling for the sake of correctness.

diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -307,6 +307,7 @@ ngx_http_do_read_client_request_body(ngx
     c = r->connection;
     rb = r->request_body;
     flush = 1;
+    n = NGX_AGAIN;
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http read client request body");
@@ -432,7 +433,7 @@ ngx_http_do_read_client_request_body(ngx
             break;
         }
 
-        if (!c->read->ready || rb->rest == 0) {
+        if (n == NGX_AGAIN || !c->read->ready || rb->rest == 0) {
 
             clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
             ngx_add_timer(c->read, clcf->client_body_timeout);



From mdounin at mdounin.ru  Fri Mar 15 18:14:21 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:21 +0300
Subject: [PATCH 07 of 10] Request body: improved c->read->ready flag handling
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <9f3e833ac0e10401d2da.1710526461@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526253 -10800
#      Fri Mar 15 21:10:53 2024 +0300
# Node ID 9f3e833ac0e10401d2da60953723d160c1bf4b5e
# Parent  3cd5f5e21b771ba547097e729642cf568ac49932
Request body: improved c->read->ready flag handling.

Previously, the c->read->ready flag was only checked when c->recv() wasn't
able to fill the whole body buffer.  Now the flag is also checked if
the buffer is fully filled.

This ensures that ev->available checks as introduced in 7583:efd71d49bde0
and 7584:9d2ad2fb4423 will be able to prevent reading from the socket
when using event methods other than kqueue or epoll, and when using SSL.

This might be important to avoid looping for a long time when reading
request body from fast clients.

diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -413,7 +413,7 @@ ngx_http_do_read_client_request_body(ngx
                 break;
             }
 
-            if (rb->buf->last < rb->buf->end) {
+            if (!c->read->ready) {
                 break;
             }
         }



From mdounin at mdounin.ru  Fri Mar 15 18:14:22 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:22 +0300
Subject: [PATCH 08 of 10] Request body: limited chunk extensions and trailer
 headers
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <ece1df3184a22241682c.1710526462@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526256 -10800
#      Fri Mar 15 21:10:56 2024 +0300
# Node ID ece1df3184a22241682cb5ee8402dbb79015b204
# Parent  9f3e833ac0e10401d2da60953723d160c1bf4b5e
Request body: limited chunk extensions and trailer headers.

Previously, arbitrary amounts of chunk extensions and trailer headers were
accepted and skipped.  Despite being under limit_conn / limit_req limits
(if configured), this can be a DoS vector, so it is now limited by the
client_max_body_size limit.

Reported by Bartek Nowotarski.

diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h
--- a/src/http/ngx_http.h
+++ b/src/http/ngx_http.h
@@ -65,6 +65,7 @@ struct ngx_http_chunked_s {
     ngx_uint_t           state;
     off_t                size;
     off_t                length;
+    off_t                skipped;
 };
 
 
diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c
--- a/src/http/ngx_http_parse.c
+++ b/src/http/ngx_http_parse.c
@@ -2257,6 +2257,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_chunk_data;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
@@ -2298,6 +2301,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_trailer;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
@@ -2333,6 +2339,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_trailer;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -1141,6 +1141,17 @@ ngx_http_request_body_chunked_filter(ngx
                 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
                 if (clcf->client_max_body_size
+                    && clcf->client_max_body_size < rb->chunked->skipped)
+                {
+                    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                  "client sent too many chunk extensions");
+
+                    r->lingering_close = 1;
+
+                    return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE;
+                }
+
+                if (clcf->client_max_body_size
                     && clcf->client_max_body_size
                        - r->headers_in.content_length_n < rb->chunked->size)
                 {
@@ -1241,6 +1252,20 @@ ngx_http_request_body_chunked_filter(ngx
 
             if (rc == NGX_AGAIN) {
 
+                clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+                if (clcf->client_max_body_size
+                    && clcf->client_max_body_size < rb->chunked->skipped)
+                {
+                    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                  "client sent too many chunk extensions "
+                                  "or trailer headers");
+
+                    r->lingering_close = 1;
+
+                    return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE;
+                }
+
                 /* set rb->rest, amount of data we want to see next time */
 
                 cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module);



From mdounin at mdounin.ru  Fri Mar 15 18:14:23 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:23 +0300
Subject: [PATCH 09 of 10] HTTP/3: synced request body reading changes to
 reduce diffs
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <9580cb3029058154973d.1710526463@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526258 -10800
#      Fri Mar 15 21:10:58 2024 +0300
# Node ID 9580cb3029058154973dd7b28c9da709c5bc3e31
# Parent  ece1df3184a22241682cb5ee8402dbb79015b204
HTTP/3: synced request body reading changes to reduce diffs.

diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c
--- a/src/http/v3/ngx_http_v3_request.c
+++ b/src/http/v3/ngx_http_v3_request.c
@@ -1329,6 +1329,7 @@ ngx_http_v3_do_read_client_request_body(
     c = r->connection;
     rb = r->request_body;
     flush = 1;
+    n = NGX_AGAIN;
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http3 read client request body");
@@ -1432,7 +1433,7 @@ ngx_http_v3_do_read_client_request_body(
                 break;
             }
 
-            if (rb->buf->last < rb->buf->end) {
+            if (!c->read->ready) {
                 break;
             }
         }
@@ -1452,7 +1453,7 @@ ngx_http_v3_do_read_client_request_body(
             break;
         }
 
-        if (!c->read->ready || rb->rest == 0) {
+        if (n == NGX_AGAIN || !c->read->ready || rb->rest == 0) {
 
             clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
             ngx_add_timer(c->read, clcf->client_body_timeout);



From mdounin at mdounin.ru  Fri Mar 15 18:14:24 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Fri, 15 Mar 2024 21:14:24 +0300
Subject: [PATCH 10 of 10] HTTP: just one empty line now accepted when parsing
 request line
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <3c67054eeb098757aaf5.1710526464@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710526261 -10800
#      Fri Mar 15 21:11:01 2024 +0300
# Node ID 3c67054eeb098757aaf59a45b3dcf38228a1552b
# Parent  9580cb3029058154973dd7b28c9da709c5bc3e31
HTTP: just one empty line now accepted when parsing request line.

This ensures that multiple CRLFs cannot be used as a DoS vector, and also
in line with RFC 9112 ("SHOULD ignore at least one empty line").  Further,
bare CRs are no longer accepted.

diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c
--- a/src/http/ngx_http_parse.c
+++ b/src/http/ngx_http_parse.c
@@ -106,6 +106,8 @@ ngx_http_parse_request_line(ngx_http_req
     u_char  c, ch, *p, *m;
     enum {
         sw_start = 0,
+        sw_newline,
+        sw_method_start,
         sw_method,
         sw_spaces_before_uri,
         sw_schema,
@@ -143,7 +145,13 @@ ngx_http_parse_request_line(ngx_http_req
         case sw_start:
             r->request_start = p;
 
-            if (ch == CR || ch == LF) {
+            if (ch == CR) {
+                state = sw_newline;
+                break;
+            }
+
+            if (ch == LF) {
+                state = sw_method_start;
                 break;
             }
 
@@ -154,6 +162,25 @@ ngx_http_parse_request_line(ngx_http_req
             state = sw_method;
             break;
 
+        case sw_newline:
+
+            if (ch == LF) {
+                state = sw_method_start;
+                break;
+            }
+
+            return NGX_HTTP_PARSE_INVALID_REQUEST;
+
+        case sw_method_start:
+            r->request_start = p;
+
+            if ((ch < 'A' || ch > 'Z') && ch != '_' && ch != '-') {
+                return NGX_HTTP_PARSE_INVALID_METHOD;
+            }
+
+            state = sw_method;
+            break;
+
         case sw_method:
             if (ch == ' ') {
                 r->method_end = p - 1;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -1623,16 +1623,6 @@ ngx_http_alloc_large_header_buffer(ngx_h
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                    "http alloc large header buffer");
 
-    if (request_line && r->state == 0) {
-
-        /* the client fills up the buffer with "\r\n" */
-
-        r->header_in->pos = r->header_in->start;
-        r->header_in->last = r->header_in->start;
-
-        return NGX_OK;
-    }
-
     old = request_line ? r->request_start : r->header_name_start;
 
     cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module);



From mdounin at mdounin.ru  Sun Mar 17 23:04:27 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 18 Mar 2024 02:04:27 +0300
Subject: [PATCH 00 of 10] read hardening
In-Reply-To: <patchbomb.1710526454@vm-bsd.mdounin.ru>
References: <patchbomb.1710526454@vm-bsd.mdounin.ru>
Message-ID: <Zfd2-9d_HFG7XRsQ@mdounin.ru>

Hello!

On Fri, Mar 15, 2024 at 09:14:14PM +0300, Maxim Dounin wrote:

> Here is a patch series which introduces various hardening in data
> reading code paths.  Review and testing appreciated.

And below are tests for the patch series.

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710640481 -10800
#      Sun Mar 17 04:54:41 2024 +0300
# Node ID 3b364d68e7fb254920b10c0aec4b3c24b0240f92
# Parent  1867428f1673214f6e2bb647a2bba0ae3a77bcb7
Tests: test for long commands with SMTP pipelining.

diff --git a/mail_smtp.t b/mail_smtp.t
--- a/mail_smtp.t
+++ b/mail_smtp.t
@@ -98,7 +98,7 @@ http {
 EOF
 
 $t->run_daemon(\&Test::Nginx::SMTP::smtp_test_daemon);
-$t->run()->plan(41);
+$t->run()->plan(43);
 
 $t->waitforsocket('127.0.0.1:' . port(8026));
 
@@ -284,6 +284,28 @@ my $s = Test::Nginx::SMTP->new();
 $s->ok('long pipelined rcpt to 4');
 $s->ok('long pipelined rset');
 
+# Pipelining longer than smtp_client_buffer, with
+# extra pipelined commands to be processed by nginx itself
+
+$s = Test::Nginx::SMTP->new(PeerAddr => '127.0.0.1:' . port(8027));
+$s->read();
+$s->send('EHLO example.com');
+$s->read();
+
+$s->send('MAIL FROM:<test at example.com> FOO=' . ('X' x 90) . CRLF
+	. 'RCPT TO:<test at example.com>' . CRLF
+	. 'RSET');
+
+$s->read();
+
+TODO: {
+local $TODO = 'not yet';
+
+$s->ok('pipelined long rcpt to');
+$s->ok('pipelined long rset');
+
+}
+
 # Connection must stay even if error returned to rcpt to command
 
 $s = Test::Nginx::SMTP->new();
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710640483 -10800
#      Sun Mar 17 04:54:43 2024 +0300
# Node ID d81108e6e92bded9f5c6ba4f66268ef5ca17a443
# Parent  3b364d68e7fb254920b10c0aec4b3c24b0240f92
Tests: mail max_commands tests.

diff --git a/mail_max_commands.t b/mail_max_commands.t
new file mode 100644
--- /dev/null
+++ b/mail_max_commands.t
@@ -0,0 +1,127 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for mail max_commands.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+use Test::Nginx::IMAP;
+use Test::Nginx::POP3;
+use Test::Nginx::SMTP;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+local $SIG{PIPE} = 'IGNORE';
+
+my $t = Test::Nginx->new()->has(qw/mail imap pop3 smtp/)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+mail {
+    auth_http  http://127.0.0.1:8080; # unused
+
+    max_commands 1;
+
+    server {
+        listen     127.0.0.1:8143;
+        protocol   imap;
+    }
+
+    server {
+        listen     127.0.0.1:8110;
+        protocol   pop3;
+    }
+
+    server {
+        listen     127.0.0.1:8025;
+        protocol   smtp;
+    }
+}
+
+EOF
+
+$t->try_run('no max_commands')->plan(18);
+
+###############################################################################
+
+# imap
+
+my $s = Test::Nginx::IMAP->new();
+$s->read();
+
+$s->send('a01 NOOP');
+$s->check(qr/^a01 OK/, 'imap first noop');
+$s->send('a02 NOOP');
+$s->check(qr/^a02 BAD/, 'imap second noop rejected');
+$s->send('a03 NOOP');
+$s->check(qr/^$/, 'imap max commands');
+
+$s = Test::Nginx::IMAP->new();
+$s->read();
+
+$s->send('a01 NOOP' . CRLF . 'a02 NOOP' . CRLF . 'a03 NOOP');
+$s->check(qr/^a01 OK/, 'imap pipelined first noop');
+$s->check(qr/^a02 BAD/, 'imap pipelined second noop rejected');
+$s->check(qr/^$/, 'imap pipelined max commands');
+
+# pop3
+
+$s = Test::Nginx::POP3->new();
+$s->read();
+
+$s->send('NOOP');
+$s->check(qr/^\+OK/, 'pop3 first noop');
+$s->send('NOOP');
+$s->check(qr/^-ERR/, 'pop3 second noop');
+$s->send('NOOP');
+$s->check(qr/^$/, 'pop3 max commands');
+
+$s = Test::Nginx::POP3->new();
+$s->read();
+
+$s->send('NOOP' . CRLF . 'NOOP' . CRLF . 'NOOP');
+$s->check(qr/^\+OK/, 'pop3 pipelined first noop');
+$s->check(qr/^-ERR/, 'pop3 pipelined second noop rejected');
+$s->check(qr/^$/, 'pop3 pipelined max commands');
+
+# smtp
+
+$s = Test::Nginx::SMTP->new();
+$s->read();
+
+$s->send('RSET');
+$s->check(qr/^2.. /, 'smtp first rset');
+$s->send('RSET');
+$s->check(qr/^5.. /, 'smtp second rset rejected');
+$s->send('RSET');
+$s->check(qr/^$/, 'smtp max commands');
+
+$s = Test::Nginx::SMTP->new();
+$s->read();
+
+$s->send('RSET' . CRLF . 'RSET' . CRLF . 'RSET');
+$s->check(qr/^2.. /, 'smtp pipelined first rset');
+$s->check(qr/^5.. /, 'smtp pipelined second rset rejected');
+$s->check(qr/^$/, 'smtp pipelined max commands');
+
+###############################################################################
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710640487 -10800
#      Sun Mar 17 04:54:47 2024 +0300
# Node ID c8100b4fc4597fc0c1c59e3751cbfbe5415d159a
# Parent  d81108e6e92bded9f5c6ba4f66268ef5ca17a443
Tests: tests for request body chunked extensions and trailers.

diff --git a/body_chunked.t b/body_chunked.t
--- a/body_chunked.t
+++ b/body_chunked.t
@@ -22,7 +22,7 @@ use Test::Nginx;
 select STDERR; $| = 1;
 select STDOUT; $| = 1;
 
-my $t = Test::Nginx->new()->has(qw/http proxy rewrite/)->plan(18);
+my $t = Test::Nginx->new()->has(qw/http proxy rewrite/)->plan(24);
 
 $t->write_file_expand('nginx.conf', <<'EOF');
 
@@ -119,6 +119,8 @@ like(http_get_body('/single', '012345678
 	qr/X-Body: (0123456789){128}\x0d?$/ms, 'body in single buffer');
 
 like(http_get_body('/large', '0123456789' x 128), qr/ 413 /, 'body too large');
+like(http_get_body('/large', 'X' x 1024), qr/ 200 /, 'body exact limit');
+like(http_get_body('/large', 'X' x 1025), qr/ 413 /, 'body just above limit');
 
 # pipelined requests
 
@@ -162,6 +164,66 @@ like(
 	qr/400 Bad/, 'runaway chunk discard'
 );
 
+# chunk extensions and trailers
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. ('1; foo' . CRLF . 'X' . CRLF) x 16
+		. '0' . CRLF . CRLF
+	),
+	qr/ 200 /, 'chunk extensions'
+);
+
+TODO: {
+local $TODO = 'not yet';
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. ('1; foo' . CRLF . 'X' . CRLF) x 512
+		. '0' . CRLF . CRLF
+	),
+	qr/ 413 /, 'too many chunk extensions'
+);
+
+}
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. '1' . CRLF . 'X' . CRLF
+		. '0' . CRLF . ('X-Trailer: foo' . CRLF) x 16 . CRLF
+	),
+	qr/ 200 /, 'trailers'
+);
+
+TODO: {
+local $TODO = 'not yet';
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. '1' . CRLF . 'X' . CRLF
+		. '0' . CRLF . ('X-Trailer: foo' . CRLF) x 512 . CRLF
+	),
+	qr/ 413 /, 'too many trailers'
+);
+
+}
+
 # proxy_next_upstream
 
 like(http_get_body('/next', '0123456789'),
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710640490 -10800
#      Sun Mar 17 04:54:50 2024 +0300
# Node ID 7ea29608aae02f2df98d5918d0bca4e2d5737129
# Parent  c8100b4fc4597fc0c1c59e3751cbfbe5415d159a
Tests: basic request parsing tests.

diff --git a/http_request.t b/http_request.t
new file mode 100644
--- /dev/null
+++ b/http_request.t
@@ -0,0 +1,190 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for basic HTTP request parsing.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+
+use Socket qw/ CRLF CR LF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()->has(qw/http rewrite/)->plan(40)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+    %%TEST_GLOBALS_HTTP%%
+
+    server {
+        listen 127.0.0.1:8080;
+        return 200 ok\n;
+    }
+}
+
+EOF
+
+$t->run();
+
+###############################################################################
+
+# some basic HTTP/0.9, HTTP/1.0, and HTTP/1.1 requests
+
+like(http(
+	"GET /" . CRLF
+), qr/^ok/s, 'http/0.9 request');
+
+like(http(
+	"GET / HTTP/1.0" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.0 request');
+
+like(http(
+	"GET / HTTP/1.0" . CRLF .
+	"Host: foo" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.0 request with host');
+
+like(http(
+	"GET / HTTP/1.1" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.1 request');
+
+like(http(
+	"GET / HTTP/1.1" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 400 /, 'http/1.1 request rejected without host');
+
+like(http(
+	"GET http://foo/ HTTP/1.1" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.1 request absolute form');
+
+# ensure an empty line is ignored before the request
+
+like(http(CRLF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'empty line ignored');
+like(http(LF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'empty line with just LF ignored');
+
+TODO: {
+local $TODO = 'not yet';
+
+like(http(CR . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'empty line with just CR rejected');
+like(http(CRLF . CRLF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple empty lines rejected');
+like(http(LF . LF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple LFs rejected');
+like(http(CR . CR . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple CRs rejected');
+
+}
+
+# method
+
+like(http("FOO / HTTP/1.0" . CRLF . CRLF), qr/ 200 /, 'method');
+like(http("FOO-BAR / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'method with dash');
+like(http("FOO_BAR / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'method with underscore');
+like(http("FOO.BAR / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'method with dot rejected');
+like(http("get / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'method in lowercase rejected');
+
+# URI
+
+like(http("GET /foo12.bar HTTP/1.0" . CRLF . CRLF), qr/ 200 /, 'uri');
+like(http("GET /control\x0d HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with CR');
+like(http("GET /control\x01 HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with control');
+like(http("GET /control\t HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with tab');
+
+# version
+
+like(http(
+	"GET / HTTP/1.2" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'version 1.2');
+
+like(http(
+	"GET / HTTP/1.99" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'version 1.99');
+
+like(http("GET / HTTP/1.000" . CRLF . CRLF), qr/ 200 /,
+	'version leading zeros');
+like(http("GET / HTTP/2.0" . CRLF . CRLF), qr/ 505 /,
+	'version too high rejected');
+like(http("GET / HTTP/1.x" . CRLF . CRLF), qr/ 400 /,
+	'version non-numeric rejected');
+like(http("GET / HTTP/1.100" . CRLF . CRLF), qr/ 400 /,
+	'version too high minor rejected');
+
+like(http("GET / http/1.0" . CRLF . CRLF), qr/ 400 /,
+	'lowercase protocol rejected');
+
+# spaces in request line
+
+like(http("GET / HTTP/1.0  " . CRLF . CRLF), qr/ 200 /,
+	'spaces after version');
+like(http("GET /   HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'spaces after uri');
+like(http("GET   / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'spaces before uri');
+
+like(http("GET / HTTP/ 1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces before version rejected');
+like(http("GET / HTTP /1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces after protocol rejected');
+like(http("GET / HT TP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces within protocol rejected');
+like(http(" GET / HTTP/ 1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces before method rejected');
+
+# headers
+
+like(http("GET / HTTP/1.0" . CRLF . "Foo: bar" . CRLF . CRLF), qr/ 200 /,
+	'header');
+like(http("GET / HTTP/1.0" . CRLF . "Foo : bar" . CRLF . CRLF), qr/ 400 /,
+	'header with space rejected');
+like(http("GET / HTTP/1.0" . CRLF . " Foo: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with leading space rejected');
+like(http("GET / HTTP/1.0" . CRLF . "Foo\x01: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with control rejected');
+like(http("GET / HTTP/1.0" . CRLF . "Foo\t: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with tab rejected');
+
+###############################################################################

-- 
Maxim Dounin
http://mdounin.ru/


From mdounin at mdounin.ru  Mon Mar 18 18:54:31 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Mon, 18 Mar 2024 21:54:31 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <21cfa15983117d64856b62197d96d3aa@elven.pw>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru>
 <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
 <21cfa15983117d64856b62197d96d3aa@elven.pw>
Message-ID: <ZfiN59wf31DoSvXw@mdounin.ru>

Hello!

On Fri, Mar 15, 2024 at 02:01:13PM +0300, Lafiel wrote:

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1708074268 -10800
> #      Fri Feb 16 12:04:28 2024 +0300
> # Node ID 24be235c0a9c883718db1e54bf742bd9be782ee4
> # Parent  89bff782528a91ad123b63b624f798e6fd9c8e68
> MIME: change type image/x-ms-bmp to image/image
> 
> For .bmp files the image/bmp mime type is used according to IANA [1].
> 
> In Apache also uses this type [2]:
> $ awk '/^image\/bmp/' mime.types
> image/bmp                                       bmp
> 
> Extension usage statistics, according to httparhcive.org [3]:
> $ awk 'NR==1||/^bmp,/' httparchive_exts.csv
> ext,total_pages,total_requests
> bmp,17013,37954
> dib,22,28
> 
> The csv file was made with the following query:
> 
> SELECT
>   ext,
>   COUNT(distinct pageid) total_pages,
>   COUNT(0) total_requests
> FROM
>   `httparchive.summary_requests.2024_01_01_desktop`
> GROUP BY
>   ext
> ORDER BY
>   total_requests DESC
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/image/bmp
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> [3] https://httparchive.org
> 
> diff -r 89bff782528a -r 24be235c0a9c conf/mime.types
> --- a/conf/mime.types   Wed Feb 14 20:03:00 2024 +0400
> +++ b/conf/mime.types   Fri Feb 16 12:04:28 2024 +0300
> @@ -16,6 +16,7 @@
>      text/x-component                                 htc;
> 
>      image/avif                                       avif;
> +    image/bmp                                        bmp;
>      image/png                                        png;
>      image/svg+xml                                    svg svgz;
>      image/tiff                                       tif tiff;
> @@ -23,7 +24,6 @@
>      image/webp                                       webp;
>      image/x-icon                                     ico;
>      image/x-jng                                      jng;
> -    image/x-ms-bmp                                   bmp;
> 
>      font/woff                                        woff;
>      font/woff2                                       woff2;

Since the bmp extension is already there, it hardly make sense to 
refer to HTTP Archive data.  Rather, I would focus on the browsers 
compatibility, as mentioned earlier in the thread.  Please take a 
look at the patch below.

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1710787295 -10800
#      Mon Mar 18 21:41:35 2024 +0300
# Node ID 27f50fb1c680b8c9589f555454d100fe3e75b024
# Parent  3c67054eeb098757aaf59a45b3dcf38228a1552b
MIME: changed type for bmp to image/bmp.

At least Chrome, Safari, and Edge do no show image/x-ms-bmp when requested
directly (and instead show a download dialog), but do so for IANA-registered
MIME type image/bmp.  Mozilla considers doing the same
(https://bugzilla.mozilla.org/show_bug.cgi?id=1422725).

Further, Apache uses image/bmp since the bmp extension was added to mime.types
in Apache 1.3.6 (and also used it at least since 1.3.0 in magic-based type
detection).

The bmp extensions with the image/x-ms-bmp type was added in 863:57cb2052dcc6
(nginx 0.4.14), as a part of a large mime.types change, and it is not clear
why image/x-ms-bmp was used instead of image/bmp.

As such, changed MIME type for bmp files to image/bmp.

Requested by Yuriy Izorkin.

diff --git a/conf/mime.types b/conf/mime.types
--- a/conf/mime.types
+++ b/conf/mime.types
@@ -16,6 +16,7 @@ types {
     text/x-component                                 htc;
 
     image/avif                                       avif;
+    image/bmp                                        bmp;
     image/png                                        png;
     image/svg+xml                                    svg svgz;
     image/tiff                                       tif tiff;
@@ -23,7 +24,6 @@ types {
     image/webp                                       webp;
     image/x-icon                                     ico;
     image/x-jng                                      jng;
-    image/x-ms-bmp                                   bmp;
 
     font/woff                                        woff;
     font/woff2                                       woff2;

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1708074518 -10800
> #      Fri Feb 16 12:08:38 2024 +0300
> # Node ID 94f80eface3b541d7b59d85fcc84d4acfca61e32
> # Parent  24be235c0a9c883718db1e54bf742bd9be782ee4
> MIME: update application/vnd.apple.mpegurl type
> 
> For .m3u and .m3u8 files the application/vnd.apple.mpegurl mime type
> is used according to IANA [1].
> 
> In Apache also uses this extensions [2]:
> $ awk '/m3u/' mime.types
> application/vnd.apple.mpegurl                   m3u8
> audio/x-mpegurl                                 m3u
> 
> There are practically no extensions in httparhcive.org statistics [3]:
> $ awk 'NR==1||/^m3u,/||/^msu8,/' httparchive_exts.csv
> ext,total_pages,total_requests
> m3u,15,302
> 
> But these extensions are often used in various streaming services.
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.apple.mpegurl
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> [3] https://httparchive.org
> 
> diff -r 24be235c0a9c -r 94f80eface3b conf/mime.types
> --- a/conf/mime.types   Fri Feb 16 12:04:28 2024 +0300
> +++ b/conf/mime.types   Fri Feb 16 12:08:38 2024 +0300
> @@ -35,7 +35,7 @@
>      application/pdf                                  pdf;
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
> -    application/vnd.apple.mpegurl                    m3u8;
> +    application/vnd.apple.mpegurl                    m3u m3u8;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
>      application/vnd.ms-excel                         xls;

Note that Apache uses audio/x-mpegurl, which is a separate type.

The m3u extension is historically used by audio playlists in the 
MP3 URL format as originally introduced Fraunhofer, see
https://en.wikipedia.org/wiki/M3U.

While Apple's playlist format is based on the original MP3 URL 
format, the are differences.  In particular, m3u files were 
expected to be in the native charset, while 
application/vnd.apple.mpegurl files are expected to be in UTF-8.  
While HLS specification (RFC 8216, independent submission, an 
informational RFC from Apple) makes no distinction between the two 
extensions, it is quite possible the distinction remains in 
practice.  Apple itself recommends m3u8 extension for HLS 
playlists 
(https://developer.apple.com/documentation/http-live-streaming/deploying-a-basic-http-live-streaming-hls-stream).

While the m3u extension might be worth adding to mime.types, the 
question of the expected type remains.  Using a vendor-specific 
type which implies different handling than a vendor-neutral well 
known format might not be what real users want/expect.

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1708074976 -10800
> #      Fri Feb 16 12:16:16 2024 +0300
> # Node ID 400a32ebdb646c03c1837704eeed83b2135da0db
> # Parent  94f80eface3b541d7b59d85fcc84d4acfca61e32
> MIME: update application/vnd.ms-excel type
> 
> For .xls, .xlm, .xla, .xlc, .xlt and .xlw files the application/vnd.ms-excel
> mime type according to MSDN blog [1]
> 
> In Apache also uses this extensions [2]:
> $ awk '/xls xlm xla xlc xlt xlw/' mime.types
> application/vnd.ms-excel                        xls xlm xla xlc xlt xlw
> 
> There are practically no extensions in httparhcive.org statistics [3]:
> $ awk 'NR==1||/^xls,/||/^xlm,/||/^xla,/||/^xlc,/||/^xlt,/||/^xlw,/||/^eps3,/' httparchive_exts.csv
> ext,total_pages,total_requests
> xls,22,38
> xlt,2,2
> 
> Link for reference:
> 
> [1] https://web.archive.org/web/20160215155254/http://blogs.msdn.com/b/vsofficedeveloper/archive/2008/05/08/office-2007-open-xml-mime-types.aspx
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> [3] https://httparchive.org
> 
> diff -r 94f80eface3b -r 400a32ebdb64 conf/mime.types
> --- a/conf/mime.types   Fri Feb 16 12:08:38 2024 +0300
> +++ b/conf/mime.types   Fri Feb 16 12:16:16 2024 +0300
> @@ -38,7 +38,7 @@
>      application/vnd.apple.mpegurl                    m3u m3u8;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
> -    application/vnd.ms-excel                         xls;
> +    application/vnd.ms-excel                         xls xlm xla xlc xlt xlw;
>      application/vnd.ms-fontobject                    eot;
>      application/vnd.ms-powerpoint                    ppt;
>      application/vnd.oasis.opendocument.graphics      odg;

The numbers provided does not support the idea of adding these 
extensions.

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1708075032 -10800
> #      Fri Feb 16 12:17:12 2024 +0300
> # Node ID 8fd3313f33fab5a342a3d0bf8fa7ddd642ff76b8
> # Parent  400a32ebdb646c03c1837704eeed83b2135da0db
> MIME: update application/vnd.ms-powerpoint type
> 
> For .pot, .ppt and .pps files the application/vnd.ms-excel mime type
> according to MSDN blog [1].
> 
> In Apache also uses this extensions [2]:
> $ awk '/ppt pps pot/' mime.types
> application/vnd.ms-powerpoint                   ppt pps pot
> 
> Extension usage statistics, according to httparhcive.org [3]:
> $ awk 'NR==1||/^pot,/||/^ppt,/||/^pps,/' httparchive_exts.csv
> ext,total_pages,total_requests
> ppt,5,7
> pot,4,5
> pps,1,3
> 
> Link for reference:
> 
> [1] https://web.archive.org/web/20160215155254/http://blogs.msdn.com/b/vsofficedeveloper/archive/2008/05/08/office-2007-open-xml-mime-types.aspx
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> [3] https://httparchive.org
> 
> diff -r 400a32ebdb64 -r 8fd3313f33fa conf/mime.types
> --- a/conf/mime.types   Fri Feb 16 12:16:16 2024 +0300
> +++ b/conf/mime.types   Fri Feb 16 12:17:12 2024 +0300
> @@ -40,7 +40,7 @@
>      application/vnd.google-earth.kmz                 kmz;
>      application/vnd.ms-excel                         xls xlm xla xlc xlt xlw;
>      application/vnd.ms-fontobject                    eot;
> -    application/vnd.ms-powerpoint                    ppt;
> +    application/vnd.ms-powerpoint                    pot ppt pps;
>      application/vnd.oasis.opendocument.graphics      odg;
>      application/vnd.oasis.opendocument.presentation  odp;
>      application/vnd.oasis.opendocument.spreadsheet   ods;

The same here.

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1708075086 -10800
> #      Fri Feb 16 12:18:06 2024 +0300
> # Node ID f35143b87faea006b9d0dae731867dfa88eecbbd
> # Parent  8fd3313f33fab5a342a3d0bf8fa7ddd642ff76b8
> MIME: update application/vnd.wap.wmlc type
> 
> For .wmlc and .wbxml files the application/vnd.wap.wmlc mime type
> according to IANA [1].
> 
> In Apache also uses this extensions [2]:
> $ awk '/application\/vnd.wap.wbxml/||/application\/vnd.wap.wmlc/' mime.types
> application/vnd.wap.wbxml                       wbxml
> application/vnd.wap.wmlc                        wmlc
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.wap.wmlc
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> 
> diff -r 8fd3313f33fa -r f35143b87fae conf/mime.types
> --- a/conf/mime.types   Fri Feb 16 12:17:12 2024 +0300
> +++ b/conf/mime.types   Fri Feb 16 12:18:06 2024 +0300
> @@ -51,7 +51,7 @@
>                                                       xlsx;
>      application/vnd.openxmlformats-officedocument.wordprocessingml.document
>                                                       docx;
> -    application/vnd.wap.wmlc                         wmlc;
> +    application/vnd.wap.wmlc                         wmlc wbxml;
>      application/wasm                                 wasm;
>      application/x-7z-compressed                      7z;
>      application/x-cocoa                              cco;

WAP in general and WML particularly are dead for a long time now, 
and I would rather consider removing the wmlc extension, as well 
as other vnd.wap types.

Further, wbxml files have their own vendor MIME type registered in 
IANA:

https://www.iana.org/assignments/media-types/application/vnd.wap.wbxml

It can be also seen in Apache mime.types as mentioned in the 
commit log.  If at all, the application/vnd.wap.wbxml type 
probably should be used for such files.  I don't see reasons 
to add wbxml files though.

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Wed Mar 20 18:13:36 2024
From: lafiel at elven.pw (Lafiel)
Date: Wed, 20 Mar 2024 21:13:36 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZfiN59wf31DoSvXw@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <21cfa15983117d64856b62197d96d3aa@elven.pw>
 <ZfiN59wf31DoSvXw@mdounin.ru>
Message-ID: <6b65a1bcf8a555b351e2f5c157589252@elven.pw>

Hello!

Maxim 
Dounin 
?????(?) 
2024-03-18 
21:54:
> Hello!
> 
> On Fri, Mar 15, 2024 at 02:01:13PM +0300, Lafiel 
> wrote:
> 
> 
> Since the bmp extension is already there, it hardly make sense 
> to
> refer to HTTP Archive data.  Rather, I would focus on the 
> browsers
> compatibility, as mentioned earlier in the thread.  Please take 
> a
> look at the patch 
> below.
> 
> ...

Okay, 
I 
agree 
with 
this 
variant.


> Note that Apache uses audio/x-mpegurl, which is a separate 
> type.
> 
> The m3u extension is historically used by audio playlists in 
> the
> MP3 URL format as originally introduced Fraunhofer, 
> see
> https://en.wikipedia.org/wiki/M3U.
> 
> While Apple's playlist format is based on the original MP3 
> URL
> format, the are differences.  In particular, m3u files 
> were
> expected to be in the native charset, 
> while
> application/vnd.apple.mpegurl files are expected to be in 
> UTF-8.
> While HLS specification (RFC 8216, independent submission, 
> an
> informational RFC from Apple) makes no distinction between the 
> two
> extensions, it is quite possible the distinction remains 
> in
> practice.  Apple itself recommends m3u8 extension for 
> HLS
> playlists
> (https://developer.apple.com/documentation/http-live-streaming/deploying-a-basic-http-live-streaming-hls-stream).
> 
> While the m3u extension might be worth adding to mime.types, 
> the
> question of the expected type remains.  Using a 
> vendor-specific
> type which implies different handling than a vendor-neutral 
> well
> known format might not be what real users 
> want/expect.

I 
think 
it's 
worth 
sticking 
with 
the 
IANA 
option 
rather 
than 
supporting
legacy 
type. 
In 
addition, 
Nginx 
did 
not 
previously 
have 
the 
m3u 
type 
and
it 
was 
identified 
as 
application/octet-stream. 
I 
don't 
think 
there 
will
be 
any 
new 
problems 
with 
application/vnd.apple.mpegurl 
type.


> The numbers provided does not support the idea of adding 
> these
> extensions.

Yes, 
but 
in 
Apache 
MS 
Excel 
and 
Powerpoint 
extensions 
are 
rare, 
but 
are 
used.
Or 
can 
shorten 
it 
to 
"xls 
xlt" 
in 
MS 
Excel 
extensions.


> WAP in general and WML particularly are dead for a long time 
> now,
> and I would rather consider removing the wmlc extension, as 
> well
> as other vnd.wap 
> types.
> 
> Further, wbxml files have their own vendor MIME type registered 
> in
> IANA:
> 
> https://www.iana.org/assignments/media-types/application/vnd.wap.wbxml
> 
> It can be also seen in Apache mime.types as mentioned in 
> the
> commit log.  If at all, the application/vnd.wap.wbxml 
> type
> probably should be used for such files.  I don't see 
> reasons
> to add wbxml files 
> though.

Then 
is 
it 
better 
to 
make 
a 
commit 
that 
removes 
this 
type, 
or 
add 
.wbxml
separately, 
like 
in 
Apache?


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240320/7cb0468c/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240320/7cb0468c/attachment.sig>

From mdounin at mdounin.ru  Thu Mar 21 20:00:06 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Thu, 21 Mar 2024 23:00:06 +0300
Subject: [nginx] MIME: changed type for bmp to image/bmp.
Message-ID: <hg.28fbf78841dc.1711051206.-8042801016281038808@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/28fbf78841dc
branches:  
changeset: 9229:28fbf78841dc
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Mar 21 22:50:11 2024 +0300
description:
MIME: changed type for bmp to image/bmp.

At least Chrome, Safari, and Edge do no show image/x-ms-bmp when requested
directly (and instead show a download dialog), but do so for IANA-registered
MIME type image/bmp.  Mozilla considers doing the same
(https://bugzilla.mozilla.org/show_bug.cgi?id=1422725).

Further, Apache uses image/bmp since the bmp extension was added to mime.types
in Apache 1.3.6 (and also used it at least since 1.3.0 in magic-based type
detection).

The bmp extensions with the image/x-ms-bmp type was added in 863:57cb2052dcc6
(nginx 0.4.14), as a part of a large mime.types change, and it is not clear
why image/x-ms-bmp was used instead of image/bmp.

As such, changed MIME type for bmp files to image/bmp.

Requested by Yuriy Izorkin.

diffstat:

 conf/mime.types |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (19 lines):

diff --git a/conf/mime.types b/conf/mime.types
--- a/conf/mime.types
+++ b/conf/mime.types
@@ -16,6 +16,7 @@ types {
     text/x-component                                 htc;
 
     image/avif                                       avif;
+    image/bmp                                        bmp;
     image/png                                        png;
     image/svg+xml                                    svg svgz;
     image/tiff                                       tif tiff;
@@ -23,7 +24,6 @@ types {
     image/webp                                       webp;
     image/x-icon                                     ico;
     image/x-jng                                      jng;
-    image/x-ms-bmp                                   bmp;
 
     font/woff                                        woff;
     font/woff2                                       woff2;


From mdounin at mdounin.ru  Thu Mar 21 20:01:30 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 21 Mar 2024 23:01:30 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <6b65a1bcf8a555b351e2f5c157589252@elven.pw>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru>
 <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
 <21cfa15983117d64856b62197d96d3aa@elven.pw>
 <ZfiN59wf31DoSvXw@mdounin.ru>
 <6b65a1bcf8a555b351e2f5c157589252@elven.pw>
Message-ID: <ZfySGgVX5ySJ6DBL@mdounin.ru>

Hello!

On Wed, Mar 20, 2024 at 09:13:36PM +0300, Lafiel wrote:

> Maxim Dounin ?????(?) 2024-03-18 21:54:
> > Hello!
> > 
> > On Fri, Mar 15, 2024 at 02:01:13PM +0300, Lafiel wrote:
> > 
> > 
> > Since the bmp extension is already there, it hardly make sense to
> > refer to HTTP Archive data.  Rather, I would focus on the browsers
> > compatibility, as mentioned earlier in the thread.  Please take a
> > look at the patch below.
> > 
> > ...
> 
> Okay, I agree with this variant.

Thanks for prodding this, committed.

> > Note that Apache uses audio/x-mpegurl, which is a separate type.
> > 
> > The m3u extension is historically used by audio playlists in the
> > MP3 URL format as originally introduced Fraunhofer, see
> > https://en.wikipedia.org/wiki/M3U.
> > 
> > While Apple's playlist format is based on the original MP3 URL
> > format, the are differences.  In particular, m3u files were
> > expected to be in the native charset, while
> > application/vnd.apple.mpegurl files are expected to be in UTF-8.
> > While HLS specification (RFC 8216, independent submission, an
> > informational RFC from Apple) makes no distinction between the two
> > extensions, it is quite possible the distinction remains in
> > practice.  Apple itself recommends m3u8 extension for HLS
> > playlists
> > (https://developer.apple.com/documentation/http-live-streaming/deploying-a-basic-http-live-streaming-hls-stream).
> > 
> > While the m3u extension might be worth adding to mime.types, the
> > question of the expected type remains.  Using a vendor-specific
> > type which implies different handling than a vendor-neutral well
> > known format might not be what real users want/expect.
> 
> I think it's worth sticking with the IANA option rather than supporting
> legacy type. In addition, Nginx did not previously have the m3u type and
> it was identified as application/octet-stream. I don't think there will
> be any new problems with application/vnd.apple.mpegurl type.

Unfortunately, IANA MIME types registry is not an universal source 
of truth for extensions to MIME types mapping.  Especially when it 
comes to vendor-specific types.  It merely lists registered MIME 
types, and extensions vendors consider to be associated with these 
types.  As long as one tries to build a mapping from extensions to 
types, there are obvious conflicts even within IANA MIME types 
registry itself (the wbxml extension is a good example), not to 
mention potential conflicts with other uses of the extensions.

And such conflicts might manifest itself even if there were no 
explicit mapping previously: if particular files work well with 
application/octet-stream type (for example, a player assumes 
correct handling based on the extension, or the file is simply 
saved by the browser and can be opened by a correct player), they 
might not work with application/vnd.apple.mpegurl (if the player 
prefers the explicit type specified over extension-based 
assumptions).

Further, in configurations where the m3u extension is explicitly 
added with correct MIME type, adding another one will cause a 
warning during configuration testing which is not possible to 
suppress.

Not to mention that distinct types can be needed to provide proper 
charsets for m3u files.

Summing the above, I would rather keep it as is, at least till 
there is some input from streaming services.  Especially keeping 
in mind that m3u8 is way more popular:

m3u8,68033,503050
m3u,15,302

> > The numbers provided does not support the idea of adding these
> > extensions.
> 
> Yes, but in Apache MS Excel and Powerpoint extensions are rare, but are
> used.
> Or can shorten it to "xls xlt" in MS Excel extensions.

You mean, based on

xls,22,38
xlt,2,2

in the HTTP Archive data?  Numbers for both xls and xlt seems very 
low, and clearly below various garbage extensions, such as 

sk%2f,1280,17999
au%2f,2983,13775
js at 6,3913,4520

While this might be a result of what HTTP Archive data are 
expected to contain (it is mostly focused on how sites are built, 
and not on various files available for download), it clearly does 
not demonstrate significance of the extensions.  There should be 
some other data to support such a change.

> > WAP in general and WML particularly are dead for a long time now,
> > and I would rather consider removing the wmlc extension, as well
> > as other vnd.wap types.
> > 
> > Further, wbxml files have their own vendor MIME type registered in
> > IANA:
> > 
> > https://www.iana.org/assignments/media-types/application/vnd.wap.wbxml
> > 
> > It can be also seen in Apache mime.types as mentioned in the
> > commit log.  If at all, the application/vnd.wap.wbxml type
> > probably should be used for such files.  I don't see reasons
> > to add wbxml files though.
> 
> Then is it better to make a commit that removes this type, or add .wbxml
> separately, like in Apache?

I'm mostly neutral on removing WAP-related types (including 
text/vnd.wap.wml, image/vnd.wap.wbmp, application/vnd.wap.wmlc, as 
well as text/vnd.wap.wml in default charset_types): while mostly 
unused now and probably safe to remove, these don't hurt much.

As for wbxml, I don't see any data supporting the idea of adding 
this extension.  If you think it's actually used, please share the 
details.

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Fri Mar 22 12:47:09 2024
From: lafiel at elven.pw (Lafiel)
Date: Fri, 22 Mar 2024 15:47:09 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZfySGgVX5ySJ6DBL@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <21cfa15983117d64856b62197d96d3aa@elven.pw>
 <ZfiN59wf31DoSvXw@mdounin.ru> <6b65a1bcf8a555b351e2f5c157589252@elven.pw>
 <ZfySGgVX5ySJ6DBL@mdounin.ru>
Message-ID: <5f8ea0ef4defdc529a63ae543ef2693b@elven.pw>

Hello!

Maxim 
Dounin 
?????(?) 
2024-03-21 
23:01:
> Unfortunately, IANA MIME types registry is not an universal 
> source
> of truth for extensions to MIME types mapping.  Especially when 
> it
> comes to vendor-specific types.  It merely lists registered 
> MIME
> types, and extensions vendors consider to be associated with 
> these
> types.  As long as one tries to build a mapping from extensions 
> to
> types, there are obvious conflicts even within IANA MIME 
> types
> registry itself (the wbxml extension is a good example), not 
> to
> mention potential conflicts with other uses of the 
> extensions.
> 
> And such conflicts might manifest itself even if there were 
> no
> explicit mapping previously: if particular files work well 
> with
> application/octet-stream type (for example, a player 
> assumes
> correct handling based on the extension, or the file is 
> simply
> saved by the browser and can be opened by a correct player), 
> they
> might not work with application/vnd.apple.mpegurl (if the 
> player
> prefers the explicit type specified over 
> extension-based
> assumptions).
> 
> Further, in configurations where the m3u extension is 
> explicitly
> added with correct MIME type, adding another one will cause 
> a
> warning during configuration testing which is not possible 
> to
> suppress.
> 
> Not to mention that distinct types can be needed to provide 
> proper
> charsets for m3u 
> files.
> 
> Summing the above, I would rather keep it as is, at least 
> till
> there is some input from streaming services.  Especially 
> keeping
> in mind that m3u8 is way more 
> popular:
> 
> m3u8,68033,503050
> m3u,15,302

Ok.


> You mean, based 
> on
> 
> xls,22,38
> xlt,2,2
> 
> in the HTTP Archive data?  Numbers for both xls and xlt seems 
> very
> low, and clearly below various garbage extensions, such 
> as
> 
> sk%2f,1280,17999
> au%2f,2983,13775
> js at 6,3913,4520
> 
> While this might be a result of what HTTP Archive data 
> are
> expected to contain (it is mostly focused on how sites are 
> built,
> and not on various files available for download), it clearly 
> does
> not demonstrate significance of the extensions.  There should 
> be
> some other data to support such a 
> change.

Yes, 
based 
on 
this 
data.
There 
is 
speculation 
that 
these 
extensions 
may 
be 
used 
more 
frequently
in 
attachments 
when 
forwarding 
emails. 
These 
types 
will 
be 
relevant 
when
viewing 
via 
the 
web 
interface. 
But 
I 
don't 
have 
such 
statistics.


> I'm mostly neutral on removing WAP-related types 
> (including
> text/vnd.wap.wml, image/vnd.wap.wbmp, application/vnd.wap.wmlc, 
> as
> well as text/vnd.wap.wml in default charset_types): while 
> mostly
> unused now and probably safe to remove, these don't hurt 
> much.
> 
> As for wbxml, I don't see any data supporting the idea of 
> adding
> this extension.  If you think it's actually used, please share 
> the
> details.

A 
quick 
internet 
search 
for 
using 
the 
wbxml 
extension 
yielded 
no 
results.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240322/ea9ceec1/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240322/ea9ceec1/attachment.sig>

From mdounin at mdounin.ru  Sat Mar 23 01:44:36 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Sat, 23 Mar 2024 04:44:36 +0300
Subject: [PATCH] Closed the radix_with_skip branch
Message-ID: <e14debe728b044587824.1711158276@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1711157445 -10800
#      Sat Mar 23 04:30:45 2024 +0300
# Branch radix_with_skip
# Node ID e14debe728b0445878248ee88bc45be93c9ea77f
# Parent  12e8e004509643e4e641da8a70dca10973a8a409
Closed the radix_with_skip branch.

The radix_with_skip branch is an archive of an experiment did in 2008,
and it is no longer relevant.  It is now closed to avoid cluttering of
the branches list.  If needed, closed branches still can be seen with
"hg branches --closed".




From mdounin at mdounin.ru  Sat Mar 23 02:03:32 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sat, 23 Mar 2024 05:03:32 +0300
Subject: [PATCH] Closed the radix_with_skip branch
In-Reply-To: <e14debe728b044587824.1711158276@vm-bsd.mdounin.ru>
References: <e14debe728b044587824.1711158276@vm-bsd.mdounin.ru>
Message-ID: <Zf44dLP16R5dRNPM@mdounin.ru>

Hello!

On Sat, Mar 23, 2024 at 04:44:36AM +0300, Maxim Dounin wrote:

> # HG changeset patch
> # User Maxim Dounin <mdounin at mdounin.ru>
> # Date 1711157445 -10800
> #      Sat Mar 23 04:30:45 2024 +0300
> # Branch radix_with_skip
> # Node ID e14debe728b0445878248ee88bc45be93c9ea77f
> # Parent  12e8e004509643e4e641da8a70dca10973a8a409
> Closed the radix_with_skip branch.
> 
> The radix_with_skip branch is an archive of an experiment did in 2008,
> and it is no longer relevant.  It is now closed to avoid cluttering of
> the branches list.  If needed, closed branches still can be seen with
> "hg branches --closed".
> 

Just in case, the changeset was generated with

hg commit --close-branch

Unfortunately, the resulting "extra: close=1" changeset 
information is not preserved in standard emailed patches, and only 
visible in log debug output (or with special templates).

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Sat Mar 23 14:31:22 2024
From: lafiel at elven.pw (Lafiel)
Date: Sat, 23 Mar 2024 17:31:22 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
Message-ID: <4943345f2efed54da9bddc75355fe1fa@elven.pw>

Hello!

Add 
new 
commits.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mime_types_02.patch
Type: text/x-diff
Size: 7878 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240323/e1062223/attachment.patch>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240323/e1062223/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240323/e1062223/attachment.sig>

From mdounin at mdounin.ru  Tue Mar 26 03:29:31 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Tue, 26 Mar 2024 06:29:31 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <4943345f2efed54da9bddc75355fe1fa@elven.pw>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru>
 <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
 <4943345f2efed54da9bddc75355fe1fa@elven.pw>
Message-ID: <ZgJBG8o4Y4y5ManS@mdounin.ru>

Hello!

On Sat, Mar 23, 2024 at 05:31:22PM +0300, Lafiel wrote:

> Hello!
> 
> Add new commits.
> 
> 
> -- 
> Best regards,
> Lafiel
> mailto:lafiel at elven.pw

> # HG changeset patch
> # User Yuriy Izorkin

Nitpicking: missing email.

> # Date 1711201515 -10800
> #      Sat Mar 23 16:45:15 2024 +0300
> # Node ID e9942c2416c92d72551ddad3a8bc3f0ea6f7751d
> # Parent  28fbf78841dc3cefb8ecc24adeab5510772d3f8c
> MIME: changed type for rar files to application/vnd.rar

Nitpicking: missing dot at the end.

> 
> The application/x-rar-compressed type is deprecated and is now
> replaced by application/vnd.rar [1].
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.rar
> 
> diff -r 28fbf78841dc -r e9942c2416c9 conf/mime.types
> --- a/conf/mime.types   Thu Mar 21 22:50:11 2024 +0300
> +++ b/conf/mime.types   Sat Mar 23 16:45:15 2024 +0300
> @@ -51,6 +51,7 @@
>                                                       xlsx;
>      application/vnd.openxmlformats-officedocument.wordprocessingml.document
>                                                       docx;
> +    application/vnd.rar                              rar;
>      application/vnd.wap.wmlc                         wmlc;
>      application/wasm                                 wasm;
>      application/x-7z-compressed                      7z;
> @@ -60,7 +61,6 @@
>      application/x-makeself                           run;
>      application/x-perl                               pl pm;
>      application/x-pilot                              prc pdb;
> -    application/x-rar-compressed                     rar;
>      application/x-redhat-package-manager             rpm;
>      application/x-sea                                sea;
>      application/x-shockwave-flash                    swf;

Overall looks good.

While Apache still uses application/x-rar-compressed, the 
registered IANA MIME type is available for 8 years now, and 
changing the type to application/vnd.rar shouldn't cause any 
issues, especially considering that expected browser behaviour for 
rar files would be to save them to disk.

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1711201810 -10800
> #      Sat Mar 23 16:50:10 2024 +0300
> # Node ID 52f7923644747a1ae80f19872a7330562fdafd04
> # Parent  e9942c2416c92d72551ddad3a8bc3f0ea6f7751d
> MIME: changed type for swf to application/vnd.adobe.flash.movie

Nitpicking: similarly to the above, missing email and missing dot.

> 
> The application/x-shockwave-flash type is considered non-standard,
> it is recommended to use application/vnd.adobe.flash.movie [1].
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.adobe.flash.movie
> 
> diff -r e9942c2416c9 -r 52f792364474 conf/mime.types
> --- a/conf/mime.types   Sat Mar 23 16:45:15 2024 +0300
> +++ b/conf/mime.types   Sat Mar 23 16:50:10 2024 +0300
> @@ -36,6 +36,7 @@
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
>      application/vnd.apple.mpegurl                    m3u8;
> +    application/vnd.adobe.flash.movie                swf;

Nitpicking: wrong sorting, should be before vnd.apple.mpegurl.

>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
>      application/vnd.ms-excel                         xls;
> @@ -63,7 +64,6 @@
>      application/x-pilot                              prc pdb;
>      application/x-redhat-package-manager             rpm;
>      application/x-sea                                sea;
> -    application/x-shockwave-flash                    swf;
>      application/x-stuffit                            sit;
>      application/x-tcl                                tcl tk;
>      application/x-x509-ca-cert                       der pem crt;

Apache still uses application/x-shockwave-flash.  Still, given 
that Flash is EoLed since 2020, this probably doesn't matter.

OTOH, it might be a good idea to don't touch it, specifically 
because it is EoLed anyway, and such a change is not expected to 
improve things, but might break some installations which still 
work.

Overall, I'm mostly neutral on the change.  Are there any specific 
reasons for the change?

> # HG changeset patch
> # User Yuriy Izorkin
> # Date 1711203285 -10800
> #      Sat Mar 23 17:14:45 2024 +0300
> # Node ID 92d1eae283c0f085cb6fa1dc8bbd2acaec6abac5
> # Parent  52f7923644747a1ae80f19872a7330562fdafd04
> MIME: changed type for pem files to application/x-x509-ca-cert

Nitpicking: similarly to the above, missing email and missing dot.

> 
> For .pem files the application/pem-certificate-chain mime type
> is used according to IANA [1]
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/x-x509-ca-cert

The link referenced does not contain anything about .pem files.
Probably should be also a link to 
https://www.iana.org/assignments/media-types/application/pem-certificate-chain 
as well.

> 
> diff -r 52f792364474 -r 92d1eae283c0 conf/mime.types
> --- a/conf/mime.types   Sat Mar 23 16:50:10 2024 +0300
> +++ b/conf/mime.types   Sat Mar 23 17:14:45 2024 +0300
> @@ -32,6 +32,7 @@
>      application/json                                 json;
>      application/mac-binhex40                         hqx;
>      application/msword                               doc;
> +    application/pem-certificate-chain                pem;
>      application/pdf                                  pdf;
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
> @@ -66,7 +67,7 @@
>      application/x-sea                                sea;
>      application/x-stuffit                            sit;
>      application/x-tcl                                tcl tk;
> -    application/x-x509-ca-cert                       der pem crt;
> +    application/x-x509-ca-cert                       der crt;
>      application/x-xpinstall                          xpi;
>      application/xhtml+xml                            xhtml;
>      application/xspf+xml                             xspf;

In practice, "crt" extension is used for PEM files with 
certificates.  As such, changing "pem" and not changing "crt" 
looks wrong.

Further, PEM-format files, which use "pem" extension, are 
certainly not just certificate chain files.  While PEM files might 
indeed contain application/pem-certificate-chain data, as defined 
in RFC 8555, that's certainly not the only option.

Overall, I would rather refrain from the change unless there are 
some better reasons than "because there is an IANA-registered 
type".

> # HG changeset patch
> # User Yuriy Izorkin

Nitpicking: missing email.

> # Date 1711203597 -10800
> #      Sat Mar 23 17:19:57 2024 +0300
> # Node ID 0d64e14efc9ccf07ea824339cab03ead3c615342
> # Parent  92d1eae283c0f085cb6fa1dc8bbd2acaec6abac5
> MIME: changed type for exe and dll files to application/vnd.microsoft.portable-executable
> For .exe and .dll files the application/vnd.microsoft.portable-executable
> mime type is used according to IANA [1].

Nitpicking: wrong commit log format, should be a summary line with 
less than 67 characters, followed by an empty line.

> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.microsoft.portable-executable
> 
> diff -r 92d1eae283c0 -r 0d64e14efc9c conf/mime.types
> --- a/conf/mime.types   Sat Mar 23 17:14:45 2024 +0300
> +++ b/conf/mime.types   Sat Mar 23 17:19:57 2024 +0300
> @@ -40,6 +40,7 @@
>      application/vnd.adobe.flash.movie                swf;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
> +    application/vnd.microsoft.portable-executable    exe dll;
>      application/vnd.ms-excel                         xls;
>      application/vnd.ms-fontobject                    eot;
>      application/vnd.ms-powerpoint                    ppt;
> @@ -73,7 +74,7 @@
>      application/xspf+xml                             xspf;
>      application/zip                                  zip;
> 
> -    application/octet-stream                         bin exe dll;
> +    application/octet-stream                         bin;
>      application/octet-stream                         deb;
>      application/octet-stream                         dmg;
>      application/octet-stream                         iso img;

Any specific reasons for the change?

While application/vnd.microsoft.portable-executable type might be 
more correct based on the IANA registration, 
application/octet-stream is expected to be at least equally good 
from practical point of view.

> # HG changeset patch
> # User Yuriy Izorkin

Nitpicking: missing email.

> # Date 1711203803 -10800
> #      Sat Mar 23 17:23:23 2024 +0300
> # Node ID a823051a9a711f5cb4d31a35ebc3a9ab31117034
> # Parent  0d64e14efc9ccf07ea824339cab03ead3c615342
> MIME: changed type for deb and udeb files to application/vnd.debian.binary-package
> For .deb and .udeb files the application/vnd.debian.binary-package mime type
> is used according to IANA [1].

Nitpicking: wrong format.

Also, the description claims that type for udeb files was changed, 
while the extension wasn't previously in mime.types.

> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.debian.binary-package
> 
> diff -r 0d64e14efc9c -r a823051a9a71 conf/mime.types
> --- a/conf/mime.types   Sat Mar 23 17:19:57 2024 +0300
> +++ b/conf/mime.types   Sat Mar 23 17:23:23 2024 +0300
> @@ -37,6 +37,7 @@
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
>      application/vnd.apple.mpegurl                    m3u8;
> +    application/vnd.debian.binary-package            deb udeb;

Nitpicking: wrong sorting, should be after vnd.adobe.flash.movie.

>      application/vnd.adobe.flash.movie                swf;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
> @@ -75,7 +76,6 @@
>      application/zip                                  zip;
> 
>      application/octet-stream                         bin;
> -    application/octet-stream                         deb;
>      application/octet-stream                         dmg;
>      application/octet-stream                         iso img;
>      application/octet-stream                         msi msp msm;

Any specific reasons for the change?

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Tue Mar 26 19:25:34 2024
From: lafiel at elven.pw (Lafiel)
Date: Tue, 26 Mar 2024 22:25:34 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZgJBG8o4Y4y5ManS@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru>
Message-ID: <a6866930770620d19f919657974efc96@elven.pw>

Hello!

Again 
I 
forgot 
to 
fix 
minor 
bugs 
in 
commits, 
I 
fixed 
it.
Any 
plans 
to 
add 
a 
git 
mirror? 
:)

Maxim 
Dounin 
?????(?) 
2024-03-26 
06:29:
> Apache still uses application/x-shockwave-flash.  Still, 
> given
> that Flash is EoLed since 2020, this probably doesn't 
> matter.
> 
> OTOH, it might be a good idea to don't touch it, 
> specifically
> because it is EoLed anyway, and such a change is not expected 
> to
> improve things, but might break some installations which 
> still
> work.
> 
> Overall, I'm mostly neutral on the change.  Are there any 
> specific
> reasons for the 
> change?

There 
are 
no 
specific 
reasons 
for 
the 
change. 
There 
is 
no 
way 
to
check 
these 
change. 
?an 
ignore 
this 
commit.


> In practice, "crt" extension is used for PEM files 
> with
> certificates.  As such, changing "pem" and not changing 
> "crt"
> looks 
> wrong.
> 
> Further, PEM-format files, which use "pem" extension, 
> are
> certainly not just certificate chain files.  While PEM files 
> might
> indeed contain application/pem-certificate-chain data, as 
> defined
> in RFC 8555, that's certainly not the only 
> option.
> 
> Overall, I would rather refrain from the change unless there 
> are
> some better reasons than "because there is an 
> IANA-registered
> type".

Added 
additional 
information 
and 
links 
in 
commit. 
Also 
added 
an
additional 
commit.


> Any specific reasons for the 
> change?
> 
> While application/vnd.microsoft.portable-executable type might 
> be
> more correct based on the IANA 
> registration,
> application/octet-stream is expected to be at least equally 
> good
> from practical point of 
> view.

Updating 
current 
MIME 
types 
:)
In 
addition, 
application/octet-stream 
already 
has 
many 
different
extensions.


> Any specific reasons for the 
> change?

Added 
additional 
information 
and 
links 
in 
commit.
In 
addition 
application/vnd.debian.binary-package 
type 
is 
used 
on 
Debian
update 
servers.
Example:
$ 
curl 
--head 
https://ftp.debian.org/debian/pool/main/n/nginx/nginx_1.24.0-2+b2_amd64.deb 
2>1 
| 
grep 
'content-type:'
content-type: 
application/vnd.debian.binary-package

I 
still 
haven't 
received 
a 
response 
regarding 
extensions 
for 
Excel
and 
PowerPoint. 
Nevertheless, 
the 
proposed 
extension 
options 
are
used 
in 
Apache.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mime_types_02.patch
Type: text/x-diff
Size: 10269 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240326/a3894c3a/attachment.patch>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240326/a3894c3a/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240326/a3894c3a/attachment.sig>

From mdounin at mdounin.ru  Thu Mar 28 00:20:21 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Thu, 28 Mar 2024 03:20:21 +0300
Subject: [nginx] Closed the radix_with_skip branch.
Message-ID: <hg.e14debe728b0.1711585221.-3284136303233486432@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/e14debe728b0
branches:  radix_with_skip
changeset: 9230:e14debe728b0
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 23 04:30:45 2024 +0300
description:
Closed the radix_with_skip branch.

The radix_with_skip branch is an archive of an experiment did in 2008,
and it is no longer relevant.  It is now closed to avoid cluttering of
the branches list.  If needed, closed branches still can be seen with
"hg branches --closed".



From mdounin at mdounin.ru  Thu Mar 28 02:20:24 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Thu, 28 Mar 2024 05:20:24 +0300
Subject: [nginx] MIME: changed type for rar files to application/vnd.rar.
Message-ID: <hg.e8b200816108.1711592424.-8059829838037843648@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/e8b200816108
branches:  
changeset: 9231:e8b200816108
user:      Yuriy Izorkin <lafiel at elven.pw>
date:      Tue Mar 26 19:39:48 2024 +0300
description:
MIME: changed type for rar files to application/vnd.rar.

The application/x-rar-compressed type is deprecated and is now
replaced by application/vnd.rar [1].

Link for reference:

[1] https://www.iana.org/assignments/media-types/application/vnd.rar

diffstat:

 conf/mime.types |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (19 lines):

diff --git a/conf/mime.types b/conf/mime.types
--- a/conf/mime.types
+++ b/conf/mime.types
@@ -51,6 +51,7 @@ types {
                                                      xlsx;
     application/vnd.openxmlformats-officedocument.wordprocessingml.document
                                                      docx;
+    application/vnd.rar                              rar;
     application/vnd.wap.wmlc                         wmlc;
     application/wasm                                 wasm;
     application/x-7z-compressed                      7z;
@@ -60,7 +61,6 @@ types {
     application/x-makeself                           run;
     application/x-perl                               pl pm;
     application/x-pilot                              prc pdb;
-    application/x-rar-compressed                     rar;
     application/x-redhat-package-manager             rpm;
     application/x-sea                                sea;
     application/x-shockwave-flash                    swf;


From mdounin at mdounin.ru  Thu Mar 28 02:20:24 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Thu, 28 Mar 2024 05:20:24 +0300
Subject: [nginx] MIME: changed type for deb files, added udeb files.
Message-ID: <hg.9a0849fd4b46.1711592425.-8059829838037843648@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/9a0849fd4b46
branches:  
changeset: 9232:9a0849fd4b46
user:      Yuriy Izorkin <lafiel at elven.pw>
date:      Tue Mar 26 21:30:08 2024 +0300
description:
MIME: changed type for deb files, added udeb files.

Changed type for .deb files to application/vnd.debian.binary-package
in accordance with man pages [1] and IANA [2].  Also added .udeb extension.

This type is also used on update servers.

Link for reference:

[1] https://manpages.debian.org/bookworm/dpkg-dev/deb.5.en.html
[2] https://www.iana.org/assignments/media-types/application/vnd.debian.binary-package

diffstat:

 conf/mime.types |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (19 lines):

diff --git a/conf/mime.types b/conf/mime.types
--- a/conf/mime.types
+++ b/conf/mime.types
@@ -36,6 +36,7 @@ types {
     application/postscript                           ps eps ai;
     application/rtf                                  rtf;
     application/vnd.apple.mpegurl                    m3u8;
+    application/vnd.debian.binary-package            deb udeb;
     application/vnd.google-earth.kml+xml             kml;
     application/vnd.google-earth.kmz                 kmz;
     application/vnd.ms-excel                         xls;
@@ -73,7 +74,6 @@ types {
     application/zip                                  zip;
 
     application/octet-stream                         bin exe dll;
-    application/octet-stream                         deb;
     application/octet-stream                         dmg;
     application/octet-stream                         iso img;
     application/octet-stream                         msi msp msm;


From mdounin at mdounin.ru  Thu Mar 28 02:20:29 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Thu, 28 Mar 2024 05:20:29 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <a6866930770620d19f919657974efc96@elven.pw>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru>
 <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
 <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru>
 <a6866930770620d19f919657974efc96@elven.pw>
Message-ID: <ZgTT7R5VkW6pkwmU@mdounin.ru>

Hello!

On Tue, Mar 26, 2024 at 10:25:34PM +0300, Lafiel wrote:

> Hello!
> 
> Again I forgot to fix minor bugs in commits, I fixed it.
> Any plans to add a git mirror? :)

It's in TODO list.

> Maxim Dounin ?????(?) 2024-03-26 06:29:
> > Apache still uses application/x-shockwave-flash.  Still, given
> > that Flash is EoLed since 2020, this probably doesn't matter.
> > 
> > OTOH, it might be a good idea to don't touch it, specifically
> > because it is EoLed anyway, and such a change is not expected to
> > improve things, but might break some installations which still
> > work.
> > 
> > Overall, I'm mostly neutral on the change.  Are there any specific
> > reasons for the change?
> 
> There are no specific reasons for the change. There is no way to
> check these change. ?an ignore this commit.

Thanks for the details.  I would rather drop it then.

> > In practice, "crt" extension is used for PEM files with
> > certificates.  As such, changing "pem" and not changing "crt"
> > looks wrong.
> > 
> > Further, PEM-format files, which use "pem" extension, are
> > certainly not just certificate chain files.  While PEM files might
> > indeed contain application/pem-certificate-chain data, as defined
> > in RFC 8555, that's certainly not the only option.
> > 
> > Overall, I would rather refrain from the change unless there are
> > some better reasons than "because there is an IANA-registered
> > type".
> 
> Added additional information and links in commit. Also added an
> additional commit.

Thanks for the details, see comments below.

> > Any specific reasons for the change?
> > 
> > While application/vnd.microsoft.portable-executable type might be
> > more correct based on the IANA registration,
> > application/octet-stream is expected to be at least equally good
> > from practical point of view.
> 
> Updating current MIME types :)
> In addition, application/octet-stream already has many different
> extensions.

Well, number of extensions mapped to application/octet-stream is 
certainly shouldn't be a deciding factor - this is essentially a 
catch-all type, suitable for most files.

Further, application/vnd.microsoft.portable-executable type looks 
questionable.  In particular, Microsoft's own IIS uses 
application/x-ms-download for "dll" files, and 
application/octet-stream for "exe" files.  Apache uses 
application/x-msdownload for both.

Based on this data, I would rather refrain from changes.  Thanks 
for trying.

> > Any specific reasons for the change?
> 
> Added additional information and links in commit.
> In addition application/vnd.debian.binary-package type is used on Debian
> update servers.
> Example:
> $ curl --head
> https://ftp.debian.org/debian/pool/main/n/nginx/nginx_1.24.0-2+b2_amd64.deb
> 2>1 | grep 'content-type:'
> content-type: application/vnd.debian.binary-package

Yep, thanks for the details, this looks like a valid enough 
reason.

> I still haven't received a response regarding extensions for Excel
> and PowerPoint. Nevertheless, the proposed extension options are
> used in Apache.

I believe we've agreed that there are no data to support inclusion 
of additional extensions.

> 
> 
> -- 
> Best regards,
> Lafiel
> mailto:lafiel at elven.pw

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711471188 -10800
> #      Tue Mar 26 19:39:48 2024 +0300
> # Node ID 60523370528bd10ca360233c48a911eb097ac04c
> # Parent  28fbf78841dc3cefb8ecc24adeab5510772d3f8c
> MIME: changed type for rar files to application/vnd.rar.
> 
> The application/x-rar-compressed type is deprecated and is now
> replaced by application/vnd.rar [1].
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.rar
> 
> diff -r 28fbf78841dc -r 60523370528b conf/mime.types
> --- a/conf/mime.types   Thu Mar 21 22:50:11 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 19:39:48 2024 +0300
> @@ -51,6 +51,7 @@
>                                                       xlsx;
>      application/vnd.openxmlformats-officedocument.wordprocessingml.document
>                                                       docx;
> +    application/vnd.rar                              rar;
>      application/vnd.wap.wmlc                         wmlc;
>      application/wasm                                 wasm;
>      application/x-7z-compressed                      7z;
> @@ -60,7 +61,6 @@
>      application/x-makeself                           run;
>      application/x-perl                               pl pm;
>      application/x-pilot                              prc pdb;
> -    application/x-rar-compressed                     rar;
>      application/x-redhat-package-manager             rpm;
>      application/x-sea                                sea;
>      application/x-shockwave-flash                    swf;

Pushed, thanks.

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711471456 -10800
> #      Tue Mar 26 19:44:16 2024 +0300
> # Node ID e51ad92ff810714aa15d394a8dc14e67e234131a
> # Parent  60523370528bd10ca360233c48a911eb097ac04c
> MIME: changed type for swf to application/vnd.adobe.flash.movie.
> 
> The application/x-shockwave-flash type is considered non-standard,
> it is recommended to use application/vnd.adobe.flash.movie [1].
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.adobe.flash.movie
> 
> diff -r 60523370528b -r e51ad92ff810 conf/mime.types
> --- a/conf/mime.types   Tue Mar 26 19:39:48 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 19:44:16 2024 +0300
> @@ -35,6 +35,7 @@
>      application/pdf                                  pdf;
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
> +    application/vnd.adobe.flash.movie                swf;
>      application/vnd.apple.mpegurl                    m3u8;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
> @@ -63,7 +64,6 @@
>      application/x-pilot                              prc pdb;
>      application/x-redhat-package-manager             rpm;
>      application/x-sea                                sea;
> -    application/x-shockwave-flash                    swf;
>      application/x-stuffit                            sit;
>      application/x-tcl                                tcl tk;
>      application/x-x509-ca-cert                       der pem crt;

Dropped this one as discussed above, thanks.

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711474326 -10800
> #      Tue Mar 26 20:32:06 2024 +0300
> # Node ID ba304001c62554448b3656777da41e6f399c524f
> # Parent  e51ad92ff810714aa15d394a8dc14e67e234131a
> MIME: changed type for pem files to application/x-x509-ca-cert.

Nitpicking: wrong summary, the change is to 
application/pem-certificate-chain.

> 
> For .pem files, the MIME type is application/pem-certificate-chain
> mime type used in accordance with section 9 of RFC 8555 [1] and
> IANA [2].
> 
> There is also an alternativevariant application/x-pem-file that is
> used in Let's Encrypt [3]. Boulder does not process Accept headers
> for Content-Type negotiation when retrieving certificates [4]. No
> matter what you pass you're going to be getting a chain of PEM
> formatted certificates [3].

It is not clear how Boulder handling of the Accept request header 
is relevant here.

Also, link [3] contains at least two comments which are directly 
relevant and contradicts to what the patch suggests, notably:

https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/2

"You are also asking about file extensions being .cer, .crt, .pem, 
but there really isn't a standardized meaning for those extensions 
or their contents. Certbot uses .pem for files that are for a 
single certificate, a chain of certificates, or a private key, and 
much other software integrating with ACME follows that convention 
as well. Windows tends to use .cer for single certificates and 
.crt for certificates intended to be used as trust anchors, but 
doesn't care if they're DER-encoded or PEM-encoded, they just have 
different default behavior when double-clicking on them (as .crt 
will default to asking if you want to add it to your trust store 
and .cer will default to just showing you, at least when I last 
tried them)."

And 
https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/4, 
which suggests mapping from extensions to MIME types.  In 
particular, it uses application/x-pem-file for pem files.

Similarly, letsencrypt.org is using application/x-pem-file for pem 
files, notably for their own keys.

> 
> Link for reference:
> 
> [1] https://www.rfc-editor.org/rfc/rfc8555#section-9
> [2] https://www.iana.org/assignments/media-types/application/x-x509-ca-cert
> [3] https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990
> [4] https://github.com/letsencrypt/boulder/blob/main/docs/acme-divergences.md#section-742
> 
> diff -r e51ad92ff810 -r ba304001c625 conf/mime.types
> --- a/conf/mime.types   Tue Mar 26 19:44:16 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 20:32:06 2024 +0300
> @@ -32,6 +32,7 @@
>      application/json                                 json;
>      application/mac-binhex40                         hqx;
>      application/msword                               doc;
> +    application/pem-certificate-chain                pem;
>      application/pdf                                  pdf;

Nitpicking: wrong sorting.

>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;
> @@ -66,7 +67,7 @@
>      application/x-sea                                sea;
>      application/x-stuffit                            sit;
>      application/x-tcl                                tcl tk;
> -    application/x-x509-ca-cert                       der pem crt;
> +    application/x-x509-ca-cert                       der crt;
>      application/x-xpinstall                          xpi;
>      application/xhtml+xml                            xhtml;
>      application/xspf+xml                             xspf;

As previously noted, I think it is utterly wrong to use different 
MIME types for "crt" and "pem" files, since these are typically 
used as synonyms for certificate files in PEM format.

Just for the record, Apache uses application/x-x509-ca-cert for 
"der" and "crt" files, and no specific type for "pem" (hence the 
default will be used).  IIS uses application/x-x509-ca-cert for 
"crt", "der", and also no specific type for "pem".

Overall, I would rather refrain from the change, at least till 
there there is a good understanding what any why we want to 
change, and how it will affect various use cases.

Dropped this for now, thanks.

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711475606 -10800
> #      Tue Mar 26 20:53:26 2024 +0300
> # Node ID 0c981a83e6de7b7ae52f8d8c17f7ee79c9355b5b
> # Parent  ba304001c62554448b3656777da41e6f399c524f
> MIME: add application/pkix-cert mime type.
> 
> Extension usage statistics, according to httparhcive.org [3]:
> $ awk 'NR==1||/^cer,/' httparchive_exts.csv
> ext,total_pages,total_requests
> cer,8998,13225
> 
> The csv file was made with the following query:
> 
> SELECT
>   ext,
>   COUNT(distinct pageid) total_pages,
>   COUNT(0) total_requests
> FROM
>   `httparchive.summary_requests.2024_01_01_desktop`
> GROUP BY
>   ext
> ORDER BY
>   total_requests DESC
> 
> In Apache also uses this type [2]:
> $ awk '/^application\/pkix-cert/' mime.types
> application/pkix-cert                           cer
> 
> Link for reference:
> 
> [1] https://httparchive.org
> [2] https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
> 
> diff -r ba304001c625 -r 0c981a83e6de conf/mime.types
> --- a/conf/mime.types   Tue Mar 26 20:32:06 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 20:53:26 2024 +0300
> @@ -33,6 +33,7 @@
>      application/mac-binhex40                         hqx;
>      application/msword                               doc;
>      application/pem-certificate-chain                pem;
> +    application/pkix-cert                            cer;
>      application/pdf                                  pdf;
>      application/postscript                           ps eps ai;
>      application/rtf                                  rtf;

(Nitpicking: wrong sorting.)

I haven't see any requests for adding this extension and type.  
While httparchive indeed suggests it is actually used, it seems to 
be way less popular than crt:

$ grep -n 'crt\|cer' httparchive_exts.csv 
33:crt,182646,209814
93:cer,8998,13225
6578:cert,3,3

Unless there are requests, I would rather refrain from adding 
it.  Also, if at all, this probably should be added along with 
"crl" from the same RFC.

Dropped this for now, thanks.

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711475987 -10800
> #      Tue Mar 26 20:59:47 2024 +0300
> # Node ID 6fe9c45cbf6fdc33ab3a48f984b5803c9576f07e
> # Parent  0c981a83e6de7b7ae52f8d8c17f7ee79c9355b5b
> MIME: changed type for exe and dll files to application/vnd.microsoft.portable-executable.
> 
> For .exe and .dll files the application/vnd.microsoft.portable-executable
> mime type is used according to IANA [1].
> 
> Link for reference:
> 
> [1] https://www.iana.org/assignments/media-types/application/vnd.microsoft.portable-executable
> 
> diff -r 0c981a83e6de -r 6fe9c45cbf6f conf/mime.types
> --- a/conf/mime.types   Tue Mar 26 20:53:26 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 20:59:47 2024 +0300
> @@ -41,6 +41,7 @@
>      application/vnd.apple.mpegurl                    m3u8;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
> +    application/vnd.microsoft.portable-executable    exe dll;
>      application/vnd.ms-excel                         xls;
>      application/vnd.ms-fontobject                    eot;
>      application/vnd.ms-powerpoint                    ppt;
> @@ -74,7 +75,7 @@
>      application/xspf+xml                             xspf;
>      application/zip                                  zip;
> 
> -    application/octet-stream                         bin exe dll;
> +    application/octet-stream                         bin;
>      application/octet-stream                         deb;
>      application/octet-stream                         dmg;
>      application/octet-stream                         iso img;

As per the discussion above, dropped this for now, thanks.

> # HG changeset patch
> # User Yuriy Izorkin <lafiel at elven.pw>
> # Date 1711477808 -10800
> #      Tue Mar 26 21:30:08 2024 +0300
> # Node ID 11dedb9aa2bc4c70f52d11c5dc6a518bc288369f
> # Parent  6fe9c45cbf6fdc33ab3a48f984b5803c9576f07e
> MIME: changed type for deb and udeb files to application/vnd.debian.binary-package.
> 
> For .deb files to application/vnd.debian.binary-package mime type used in accordance
> with man pages [1] and IANA [2]. Also added .udeb extension.
> 
> This type is also used on update servers.
> 
> Link for reference:
> 
> [1] https://manpages.debian.org/bookworm/dpkg-dev/deb.5.en.html
> [2] https://www.iana.org/assignments/media-types/application/vnd.debian.binary-package
> 
> diff -r 6fe9c45cbf6f -r 11dedb9aa2bc conf/mime.types
> --- a/conf/mime.types   Tue Mar 26 20:59:47 2024 +0300
> +++ b/conf/mime.types   Tue Mar 26 21:30:08 2024 +0300
> @@ -39,6 +39,7 @@
>      application/rtf                                  rtf;
>      application/vnd.adobe.flash.movie                swf;
>      application/vnd.apple.mpegurl                    m3u8;
> +    application/vnd.debian.binary-package            deb udeb;
>      application/vnd.google-earth.kml+xml             kml;
>      application/vnd.google-earth.kmz                 kmz;
>      application/vnd.microsoft.portable-executable    exe dll;
> @@ -76,7 +77,6 @@
>      application/zip                                  zip;
> 
>      application/octet-stream                         bin;
> -    application/octet-stream                         deb;
>      application/octet-stream                         dmg;
>      application/octet-stream                         iso img;
>      application/octet-stream                         msi msp msm;

Pushed with minor commit log cleanup, thanks.

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Fri Mar 29 17:25:18 2024
From: lafiel at elven.pw (Lafiel)
Date: Fri, 29 Mar 2024 20:25:18 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZgTT7R5VkW6pkwmU@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru> <a6866930770620d19f919657974efc96@elven.pw>
 <ZgTT7R5VkW6pkwmU@mdounin.ru>
Message-ID: <bd554a2d48fad3f2f95bebdfc3dbbeb7@elven.pw>

Hello!

Maxim 
Dounin 
?????(?) 
2024-03-28 
05:20:
> It's in TODO 
> list.

Then 
should 
take 
a 
closer 
look 
at 
Forgejo. 
It 
is 
developing 
a 
new
ForgeFed 
protocol 
for 
collaborative 
development:
https://forgefed.org


> Well, number of extensions mapped to application/octet-stream 
> is
> certainly shouldn't be a deciding factor - this is essentially 
> a
> catch-all type, suitable for most 
> files.

The 
configuration 
file 
`conf/nginx.conf` 
contains 
parameter
`default_type 
application/octet-stream;`. 
It 
is 
entirely 
possible
to 
change 
it 
to 
default 
instead 
of 
`text/plain` 
and 
remove 
remaining
`application/octet-stream` 
types 
from 
the 
mime.types 
file.
Small 
optimization 
:)


> Further, application/vnd.microsoft.portable-executable type 
> looks
> questionable.  In particular, Microsoft's own IIS 
> uses
> application/x-ms-download for "dll" files, 
> and
> application/octet-stream for "exe" files.  Apache 
> uses
> application/x-msdownload for 
> both.
> 
> Based on this data, I would rather refrain from changes.  
> Thanks
> for 
> trying.

I 
checked 
that 
they 
use 
`application/octet-stream` 
type 
on 
their 
servers:
$ 
curl 
--head 
https://download.microsoft.com/download/1/7/1/1718CCC4-6315-4D8E-9543-8E28A4E18C4C/dxwebsetup.exe 
2>1 
| 
grep 
'Content-Type:'
Content-Type: 
application/octet-stream


> It is not clear how Boulder handling of the Accept request 
> header
> is relevant 
> here.
> 
> Also, link [3] contains at least two comments which are 
> directly
> relevant and contradicts to what the patch suggests, 
> notably:
> 
> https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/2
> 
> "You are also asking about file extensions being .cer, .crt, 
> .pem,
> but there really isn't a standardized meaning for those 
> extensions
> or their contents. Certbot uses .pem for files that are for 
> a
> single certificate, a chain of certificates, or a private key, 
> and
> much other software integrating with ACME follows that 
> convention
> as well. Windows tends to use .cer for single certificates 
> and
> .crt for certificates intended to be used as trust anchors, 
> but
> doesn't care if they're DER-encoded or PEM-encoded, they just 
> have
> different default behavior when double-clicking on them (as 
> .crt
> will default to asking if you want to add it to your trust 
> store
> and .cer will default to just showing you, at least when I 
> last
> tried 
> them)."
> 
> And
> https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/4,
> which suggests mapping from extensions to MIME types.  
> In
> particular, it uses application/x-pem-file for pem 
> files.
> 
> Similarly, letsencrypt.org is using application/x-pem-file for 
> pem
> files, notably for their own 
> keys.

I 
wanted 
to 
give 
as 
an 
example 
that 
changing 
type 
to 
another 
should 
not 
affect 
the 
work.


> Unless there are requests, I would rather refrain from 
> adding
> it.  Also, if at all, this probably should be added along 
> with
> "crl" from the same 
> RFC.
> 
> Dropped this for now, 
> thanks.

There 
is 
no 
.crl 
extension 
in 
httparchive 
statistics, 
so 
I 
didn?t 
add 
it.
I 
can 
try 
updating 
the 
commit 
:)


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240329/2ee13b03/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240329/2ee13b03/attachment.sig>

From mdounin at mdounin.ru  Sat Mar 30 04:59:45 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:45 +0300
Subject: [nginx] Mail: switched to posted events when resuming reading.
Message-ID: <hg.9ca12c957304.1711774785.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/9ca12c957304
branches:  
changeset: 9233:9ca12c957304
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:04:25 2024 +0300
description:
Mail: switched to posted events when resuming reading.

When resuming reading after ngx_mail_send(), switched to using posted
events instead of a direct event handler call.  This ensures limited
stack usage when processing multiple pipelined commands.

diffstat:

 src/mail/ngx_mail_handler.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff --git a/src/mail/ngx_mail_handler.c b/src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c
+++ b/src/mail/ngx_mail_handler.c
@@ -799,7 +799,7 @@ ngx_mail_send(ngx_event_t *wev)
         }
 
         if (s->blocked) {
-            c->read->handler(c->read);
+            ngx_post_event(c->read, &ngx_posted_events);
         }
 
         return;


From mdounin at mdounin.ru  Sat Mar 30 04:59:45 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:45 +0300
Subject: [nginx] Mail: handling of read buffer filled with commands.
Message-ID: <hg.c690a902bfec.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/c690a902bfec
branches:  
changeset: 9234:c690a902bfec
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:05:05 2024 +0300
description:
Mail: handling of read buffer filled with commands.

If the whole read buffer was filled with commands during authentication,
exactly matching the buffer boundary, this did not cause the "client sent
too long command" error, but might result in read events being disabled
for the connection, notably with level-triggered event methods, leading
to a connection hang.

Fix is to ensure that s->blocked is set in case of c->read->ready, so
command reading is resumed after sending responses to previous commands.

diffstat:

 src/mail/ngx_mail_imap_handler.c |  2 +-
 src/mail/ngx_mail_pop3_handler.c |  2 +-
 src/mail/ngx_mail_smtp_handler.c |  2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diffs (36 lines):

diff --git a/src/mail/ngx_mail_imap_handler.c b/src/mail/ngx_mail_imap_handler.c
--- a/src/mail/ngx_mail_imap_handler.c
+++ b/src/mail/ngx_mail_imap_handler.c
@@ -226,7 +226,7 @@ ngx_mail_imap_auth_state(ngx_event_t *re
         ngx_str_set(&s->out, imap_next);
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 
diff --git a/src/mail/ngx_mail_pop3_handler.c b/src/mail/ngx_mail_pop3_handler.c
--- a/src/mail/ngx_mail_pop3_handler.c
+++ b/src/mail/ngx_mail_pop3_handler.c
@@ -262,7 +262,7 @@ ngx_mail_pop3_auth_state(ngx_event_t *re
         }
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 
diff --git a/src/mail/ngx_mail_smtp_handler.c b/src/mail/ngx_mail_smtp_handler.c
--- a/src/mail/ngx_mail_smtp_handler.c
+++ b/src/mail/ngx_mail_smtp_handler.c
@@ -550,7 +550,7 @@ ngx_mail_smtp_auth_state(ngx_event_t *re
         }
     }
 
-    if (s->buffer->pos < s->buffer->last) {
+    if (s->buffer->pos < s->buffer->last || c->read->ready) {
         s->blocked = 1;
     }
 


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Mail: handling of pipelined commands which cross buffer ...
Message-ID: <hg.d7fd0acdea64.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/d7fd0acdea64
branches:  
changeset: 9235:d7fd0acdea64
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:05:31 2024 +0300
description:
Mail: handling of pipelined commands which cross buffer boundary.

Previously, pipelined commands which cross buffer boundary were rejected
as too long, even if the command itself was short enough to be handled
within the client buffer.

Fix is to move non-processed commands to the start of the buffer after
the previous command is fully processed.

diffstat:

 src/mail/ngx_mail_imap_handler.c |  7 +++++++
 src/mail/ngx_mail_pop3_handler.c |  7 +++++++
 src/mail/ngx_mail_smtp_handler.c |  7 +++++++
 3 files changed, 21 insertions(+), 0 deletions(-)

diffs (72 lines):

diff --git a/src/mail/ngx_mail_imap_handler.c b/src/mail/ngx_mail_imap_handler.c
--- a/src/mail/ngx_mail_imap_handler.c
+++ b/src/mail/ngx_mail_imap_handler.c
@@ -102,6 +102,7 @@ void
 ngx_mail_imap_auth_state(ngx_event_t *rev)
 {
     u_char              *p;
+    size_t               n;
     ngx_int_t            rc;
     ngx_uint_t           tag;
     ngx_connection_t    *c;
@@ -285,6 +286,12 @@ ngx_mail_imap_auth_state(ngx_event_t *re
             if (s->buffer->pos == s->buffer->last) {
                 s->buffer->pos = s->buffer->start;
                 s->buffer->last = s->buffer->start;
+
+            } else {
+                n = s->buffer->last - s->buffer->pos;
+                ngx_memmove(s->buffer->start, s->buffer->pos, n);
+                s->buffer->pos = s->buffer->start;
+                s->buffer->last = s->buffer->start + n;
             }
 
             s->tag.len = 0;
diff --git a/src/mail/ngx_mail_pop3_handler.c b/src/mail/ngx_mail_pop3_handler.c
--- a/src/mail/ngx_mail_pop3_handler.c
+++ b/src/mail/ngx_mail_pop3_handler.c
@@ -119,6 +119,7 @@ ngx_mail_pop3_init_protocol(ngx_event_t 
 void
 ngx_mail_pop3_auth_state(ngx_event_t *rev)
 {
+    size_t               n;
     ngx_int_t            rc;
     ngx_connection_t    *c;
     ngx_mail_session_t  *s;
@@ -291,6 +292,12 @@ ngx_mail_pop3_auth_state(ngx_event_t *re
         if (s->buffer->pos == s->buffer->last) {
             s->buffer->pos = s->buffer->start;
             s->buffer->last = s->buffer->start;
+
+        } else {
+            n = s->buffer->last - s->buffer->pos;
+            ngx_memmove(s->buffer->start, s->buffer->pos, n);
+            s->buffer->pos = s->buffer->start;
+            s->buffer->last = s->buffer->start + n;
         }
 
         if (s->state) {
diff --git a/src/mail/ngx_mail_smtp_handler.c b/src/mail/ngx_mail_smtp_handler.c
--- a/src/mail/ngx_mail_smtp_handler.c
+++ b/src/mail/ngx_mail_smtp_handler.c
@@ -430,6 +430,7 @@ ngx_mail_smtp_create_buffer(ngx_mail_ses
 void
 ngx_mail_smtp_auth_state(ngx_event_t *rev)
 {
+    size_t               n;
     ngx_int_t            rc;
     ngx_connection_t    *c;
     ngx_mail_session_t  *s;
@@ -577,6 +578,12 @@ ngx_mail_smtp_auth_state(ngx_event_t *re
         if (s->buffer->pos == s->buffer->last) {
             s->buffer->pos = s->buffer->start;
             s->buffer->last = s->buffer->start;
+
+        } else {
+            n = s->buffer->last - s->buffer->pos;
+            ngx_memmove(s->buffer->start, s->buffer->pos, n);
+            s->buffer->pos = s->buffer->start;
+            s->buffer->last = s->buffer->start + n;
         }
 
         if (s->state) {


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Mail: max_commands directive.
Message-ID: <hg.d9a52ebb9b00.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/d9a52ebb9b00
branches:  
changeset: 9236:d9a52ebb9b00
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:05:53 2024 +0300
description:
Mail: max_commands directive.

The directive specifies the maximum number of commands allowed during
authentication, after which the connection is closed.  The default
limit is 1000, which is not expected to affect any well-behaving clients,
since authentication usually requires at most several commands, though
will effectively stop malicious clients from flooding the server with
with commands.

diffstat:

 src/mail/ngx_mail.h             |   2 ++
 src/mail/ngx_mail_core_module.c |   9 +++++++++
 src/mail/ngx_mail_handler.c     |  12 ++++++++++++
 3 files changed, 23 insertions(+), 0 deletions(-)

diffs (74 lines):

diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h
+++ b/src/mail/ngx_mail.h
@@ -116,6 +116,7 @@ typedef struct {
     ngx_msec_t              resolver_timeout;
 
     ngx_uint_t              max_errors;
+    ngx_uint_t              max_commands;
 
     ngx_str_t               server_name;
 
@@ -234,6 +235,7 @@ typedef struct {
     ngx_array_t             args;
 
     ngx_uint_t              errors;
+    ngx_uint_t              commands;
     ngx_uint_t              login_attempt;
 
     /* used to parse POP3/IMAP/SMTP command */
diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c
+++ b/src/mail/ngx_mail_core_module.c
@@ -92,6 +92,13 @@ static ngx_command_t  ngx_mail_core_comm
       offsetof(ngx_mail_core_srv_conf_t, max_errors),
       NULL },
 
+    { ngx_string("max_commands"),
+      NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_MAIL_SRV_CONF_OFFSET,
+      offsetof(ngx_mail_core_srv_conf_t, max_commands),
+      NULL },
+
       ngx_null_command
 };
 
@@ -171,6 +178,7 @@ ngx_mail_core_create_srv_conf(ngx_conf_t
     cscf->resolver_timeout = NGX_CONF_UNSET_MSEC;
 
     cscf->max_errors = NGX_CONF_UNSET_UINT;
+    cscf->max_commands = NGX_CONF_UNSET_UINT;
 
     cscf->resolver = NGX_CONF_UNSET_PTR;
 
@@ -192,6 +200,7 @@ ngx_mail_core_merge_srv_conf(ngx_conf_t 
                               30000);
 
     ngx_conf_merge_uint_value(conf->max_errors, prev->max_errors, 5);
+    ngx_conf_merge_uint_value(conf->max_commands, prev->max_commands, 1000);
 
     ngx_conf_merge_str_value(conf->server_name, prev->server_name, "");
 
diff --git a/src/mail/ngx_mail_handler.c b/src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c
+++ b/src/mail/ngx_mail_handler.c
@@ -896,6 +896,18 @@ ngx_mail_read_command(ngx_mail_session_t
         return NGX_ERROR;
     }
 
+    s->commands++;
+
+    if (s->commands > cscf->max_commands) {
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "client sent too many commands");
+
+        s->quit = 1;
+
+        return NGX_MAIL_PARSE_INVALID_COMMAND;
+    }
+
     return NGX_OK;
 }
 


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Upstream: improved c->read->ready flag handling.
Message-ID: <hg.41db21d1ca7c.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/41db21d1ca7c
branches:  
changeset: 9237:41db21d1ca7c
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:06:15 2024 +0300
description:
Upstream: improved c->read->ready flag handling.

Previously, reading the upstream response headers did not check the
c->read->ready flag.  Now the flag is checked.

This ensures that ev->available checks as introduced in 7583:efd71d49bde0
and 7584:9d2ad2fb4423 will be able to prevent reading from the socket
when using event methods other than kqueue or epoll, and when using SSL.

This might be important to avoid looping for a long time when working
with fast upstream servers over protocols where large chunks of data can
be skipped while reading response headers, notably FastCGI and gRPC.

diffstat:

 src/http/ngx_http_upstream.c |  7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)

diffs (17 lines):

diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -2437,7 +2437,12 @@ ngx_http_upstream_process_header(ngx_htt
 
     for ( ;; ) {
 
-        n = c->recv(c, u->buffer.last, u->buffer.end - u->buffer.last);
+        if (c->read->ready) {
+            n = c->recv(c, u->buffer.last, u->buffer.end - u->buffer.last);
+
+        } else {
+            n = NGX_AGAIN;
+        }
 
         if (n == NGX_AGAIN) {
 #if 0


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Request body: explicit handling of NGX_AGAIN.
Message-ID: <hg.392e8e2fd22a.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/392e8e2fd22a
branches:  
changeset: 9238:392e8e2fd22a
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:06:59 2024 +0300
description:
Request body: explicit handling of NGX_AGAIN.

Request body reading indirectly uses the "do { c->recv() } while
(c->read->ready)" form, which is not really correct, as for example
with SSL c->read->ready may be still set when c->recv() returns NGX_AGAIN
due to SSL_ERROR_WANT_WRITE (see 7351:2b5528023f6b), and therefore this
form might be an infinite loop.

Added explicit NGX_AGAIN handling for the sake of correctness.

diffstat:

 src/http/ngx_http_request_body.c |  3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diffs (20 lines):

diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -307,6 +307,7 @@ ngx_http_do_read_client_request_body(ngx
     c = r->connection;
     rb = r->request_body;
     flush = 1;
+    n = NGX_AGAIN;
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http read client request body");
@@ -432,7 +433,7 @@ ngx_http_do_read_client_request_body(ngx
             break;
         }
 
-        if (!c->read->ready || rb->rest == 0) {
+        if (n == NGX_AGAIN || !c->read->ready || rb->rest == 0) {
 
             clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
             ngx_add_timer(c->read, clcf->client_body_timeout);


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Request body: improved c->read->ready flag handling.
Message-ID: <hg.b2e16e8639c8.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/b2e16e8639c8
branches:  
changeset: 9239:b2e16e8639c8
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:09:12 2024 +0300
description:
Request body: improved c->read->ready flag handling.

Previously, the c->read->ready flag was only checked when c->recv() wasn't
able to fill the whole body buffer.  Now the flag is also checked if
the buffer is fully filled.

This ensures that ev->available checks as introduced in 7583:efd71d49bde0
and 7584:9d2ad2fb4423 will be able to prevent reading from the socket
when using event methods other than kqueue or epoll, and when using SSL.

This might be important to avoid looping for a long time when reading
request body from fast clients.

diffstat:

 src/http/ngx_http_request_body.c |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diffs (12 lines):

diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -413,7 +413,7 @@ ngx_http_do_read_client_request_body(ngx
                 break;
             }
 
-            if (rb->buf->last < rb->buf->end) {
+            if (!c->read->ready) {
                 break;
             }
         }


From mdounin at mdounin.ru  Sat Mar 30 04:59:46 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:46 +0300
Subject: [nginx] Request body: limited chunk extensions and trailer headers.
Message-ID: <hg.f3df785649ae.1711774786.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/f3df785649ae
branches:  
changeset: 9240:f3df785649ae
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:09:35 2024 +0300
description:
Request body: limited chunk extensions and trailer headers.

Previously, arbitrary amounts of chunk extensions and trailer headers were
accepted and skipped.  Despite being under limit_conn / limit_req limits
(if configured), this can be a DoS vector, so it is now limited by the
client_max_body_size limit.

Reported by Bartek Nowotarski.

diffstat:

 src/http/ngx_http.h              |   1 +
 src/http/ngx_http_parse.c        |   9 +++++++++
 src/http/ngx_http_request_body.c |  25 +++++++++++++++++++++++++
 3 files changed, 35 insertions(+), 0 deletions(-)

diffs (86 lines):

diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h
--- a/src/http/ngx_http.h
+++ b/src/http/ngx_http.h
@@ -65,6 +65,7 @@ struct ngx_http_chunked_s {
     ngx_uint_t           state;
     off_t                size;
     off_t                length;
+    off_t                skipped;
 };
 
 
diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c
--- a/src/http/ngx_http_parse.c
+++ b/src/http/ngx_http_parse.c
@@ -2257,6 +2257,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_chunk_data;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
@@ -2298,6 +2301,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_trailer;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
@@ -2333,6 +2339,9 @@ ngx_http_parse_chunked(ngx_http_request_
                 break;
             case LF:
                 state = sw_trailer;
+                break;
+            default:
+                ctx->skipped++;
             }
             break;
 
diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -1141,6 +1141,17 @@ ngx_http_request_body_chunked_filter(ngx
                 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
                 if (clcf->client_max_body_size
+                    && clcf->client_max_body_size < rb->chunked->skipped)
+                {
+                    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                  "client sent too many chunk extensions");
+
+                    r->lingering_close = 1;
+
+                    return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE;
+                }
+
+                if (clcf->client_max_body_size
                     && clcf->client_max_body_size
                        - r->headers_in.content_length_n < rb->chunked->size)
                 {
@@ -1241,6 +1252,20 @@ ngx_http_request_body_chunked_filter(ngx
 
             if (rc == NGX_AGAIN) {
 
+                clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+                if (clcf->client_max_body_size
+                    && clcf->client_max_body_size < rb->chunked->skipped)
+                {
+                    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                                  "client sent too many chunk extensions "
+                                  "or trailer headers");
+
+                    r->lingering_close = 1;
+
+                    return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE;
+                }
+
                 /* set rb->rest, amount of data we want to see next time */
 
                 cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module);


From mdounin at mdounin.ru  Sat Mar 30 04:59:47 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:47 +0300
Subject: [nginx] HTTP/3: synced request body reading changes to reduce di...
Message-ID: <hg.07ca679842de.1711774787.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/07ca679842de
branches:  
changeset: 9241:07ca679842de
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:10:24 2024 +0300
description:
HTTP/3: synced request body reading changes to reduce diffs.

diffstat:

 src/http/v3/ngx_http_v3_request.c |  5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diffs (29 lines):

diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c
--- a/src/http/v3/ngx_http_v3_request.c
+++ b/src/http/v3/ngx_http_v3_request.c
@@ -1329,6 +1329,7 @@ ngx_http_v3_do_read_client_request_body(
     c = r->connection;
     rb = r->request_body;
     flush = 1;
+    n = NGX_AGAIN;
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http3 read client request body");
@@ -1432,7 +1433,7 @@ ngx_http_v3_do_read_client_request_body(
                 break;
             }
 
-            if (rb->buf->last < rb->buf->end) {
+            if (!c->read->ready) {
                 break;
             }
         }
@@ -1452,7 +1453,7 @@ ngx_http_v3_do_read_client_request_body(
             break;
         }
 
-        if (!c->read->ready || rb->rest == 0) {
+        if (n == NGX_AGAIN || !c->read->ready || rb->rest == 0) {
 
             clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
             ngx_add_timer(c->read, clcf->client_body_timeout);


From mdounin at mdounin.ru  Sat Mar 30 04:59:47 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 07:59:47 +0300
Subject: [nginx] HTTP: just one empty line now accepted when parsing requ...
Message-ID: <hg.ddcedfa3a809.1711774787.-5636086173641214431@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx/rev/ddcedfa3a809
branches:  
changeset: 9242:ddcedfa3a809
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 05:10:40 2024 +0300
description:
HTTP: just one empty line now accepted when parsing request line.

This ensures that multiple CRLFs cannot be used as a DoS vector, and also
in line with RFC 9112 ("SHOULD ignore at least one empty line").  Further,
bare CRs are no longer accepted.

diffstat:

 src/http/ngx_http_parse.c   |  29 ++++++++++++++++++++++++++++-
 src/http/ngx_http_request.c |  10 ----------
 2 files changed, 28 insertions(+), 11 deletions(-)

diffs (73 lines):

diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c
--- a/src/http/ngx_http_parse.c
+++ b/src/http/ngx_http_parse.c
@@ -106,6 +106,8 @@ ngx_http_parse_request_line(ngx_http_req
     u_char  c, ch, *p, *m;
     enum {
         sw_start = 0,
+        sw_newline,
+        sw_method_start,
         sw_method,
         sw_spaces_before_uri,
         sw_schema,
@@ -143,7 +145,13 @@ ngx_http_parse_request_line(ngx_http_req
         case sw_start:
             r->request_start = p;
 
-            if (ch == CR || ch == LF) {
+            if (ch == CR) {
+                state = sw_newline;
+                break;
+            }
+
+            if (ch == LF) {
+                state = sw_method_start;
                 break;
             }
 
@@ -154,6 +162,25 @@ ngx_http_parse_request_line(ngx_http_req
             state = sw_method;
             break;
 
+        case sw_newline:
+
+            if (ch == LF) {
+                state = sw_method_start;
+                break;
+            }
+
+            return NGX_HTTP_PARSE_INVALID_REQUEST;
+
+        case sw_method_start:
+            r->request_start = p;
+
+            if ((ch < 'A' || ch > 'Z') && ch != '_' && ch != '-') {
+                return NGX_HTTP_PARSE_INVALID_METHOD;
+            }
+
+            state = sw_method;
+            break;
+
         case sw_method:
             if (ch == ' ') {
                 r->method_end = p - 1;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -1623,16 +1623,6 @@ ngx_http_alloc_large_header_buffer(ngx_h
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                    "http alloc large header buffer");
 
-    if (request_line && r->state == 0) {
-
-        /* the client fills up the buffer with "\r\n" */
-
-        r->header_in->pos = r->header_in->start;
-        r->header_in->last = r->header_in->start;
-
-        return NGX_OK;
-    }
-
     old = request_line ? r->request_start : r->header_name_start;
 
     cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module);


From mdounin at mdounin.ru  Sat Mar 30 05:08:35 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 08:08:35 +0300
Subject: [nginx-tests] Tests: test for long commands with SMTP pipelining.
Message-ID: <hg.92d90cc5f5e5.1711775315.-1911883231343982053@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx-tests/rev/92d90cc5f5e5
branches:  
changeset: 1952:92d90cc5f5e5
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 07:18:55 2024 +0300
description:
Tests: test for long commands with SMTP pipelining.

diffstat:

 mail_smtp.t |  24 +++++++++++++++++++++++-
 1 files changed, 23 insertions(+), 1 deletions(-)

diffs (41 lines):

diff --git a/mail_smtp.t b/mail_smtp.t
--- a/mail_smtp.t
+++ b/mail_smtp.t
@@ -98,7 +98,7 @@ http {
 EOF
 
 $t->run_daemon(\&Test::Nginx::SMTP::smtp_test_daemon);
-$t->run()->plan(41);
+$t->run()->plan(43);
 
 $t->waitforsocket('127.0.0.1:' . port(8026));
 
@@ -284,6 +284,28 @@ my $s = Test::Nginx::SMTP->new();
 $s->ok('long pipelined rcpt to 4');
 $s->ok('long pipelined rset');
 
+# Pipelining longer than smtp_client_buffer, with
+# extra pipelined commands to be processed by nginx itself
+
+$s = Test::Nginx::SMTP->new(PeerAddr => '127.0.0.1:' . port(8027));
+$s->read();
+$s->send('EHLO example.com');
+$s->read();
+
+$s->send('MAIL FROM:<test at example.com> FOO=' . ('X' x 90) . CRLF
+	. 'RCPT TO:<test at example.com>' . CRLF
+	. 'RSET');
+
+$s->read();
+
+TODO: {
+local $TODO = 'not yet' unless $t->has_version('1.25.5');
+
+$s->ok('pipelined long rcpt to');
+$s->ok('pipelined long rset');
+
+}
+
 # Connection must stay even if error returned to rcpt to command
 
 $s = Test::Nginx::SMTP->new();


From mdounin at mdounin.ru  Sat Mar 30 05:08:35 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 08:08:35 +0300
Subject: [nginx-tests] Tests: mail max_commands tests.
Message-ID: <hg.33b54de4ee23.1711775315.-1911883231343982053@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx-tests/rev/33b54de4ee23
branches:  
changeset: 1953:33b54de4ee23
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 07:19:11 2024 +0300
description:
Tests: mail max_commands tests.

diffstat:

 mail_max_commands.t |  127 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 127 insertions(+), 0 deletions(-)

diffs (132 lines):

diff --git a/mail_max_commands.t b/mail_max_commands.t
new file mode 100644
--- /dev/null
+++ b/mail_max_commands.t
@@ -0,0 +1,127 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for mail max_commands.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+use Test::Nginx::IMAP;
+use Test::Nginx::POP3;
+use Test::Nginx::SMTP;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+local $SIG{PIPE} = 'IGNORE';
+
+my $t = Test::Nginx->new()->has(qw/mail imap pop3 smtp/)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+mail {
+    auth_http  http://127.0.0.1:8080; # unused
+
+    max_commands 1;
+
+    server {
+        listen     127.0.0.1:8143;
+        protocol   imap;
+    }
+
+    server {
+        listen     127.0.0.1:8110;
+        protocol   pop3;
+    }
+
+    server {
+        listen     127.0.0.1:8025;
+        protocol   smtp;
+    }
+}
+
+EOF
+
+$t->try_run('no max_commands')->plan(18);
+
+###############################################################################
+
+# imap
+
+my $s = Test::Nginx::IMAP->new();
+$s->read();
+
+$s->send('a01 NOOP');
+$s->check(qr/^a01 OK/, 'imap first noop');
+$s->send('a02 NOOP');
+$s->check(qr/^a02 BAD/, 'imap second noop rejected');
+$s->send('a03 NOOP');
+$s->check(qr/^$/, 'imap max commands');
+
+$s = Test::Nginx::IMAP->new();
+$s->read();
+
+$s->send('a01 NOOP' . CRLF . 'a02 NOOP' . CRLF . 'a03 NOOP');
+$s->check(qr/^a01 OK/, 'imap pipelined first noop');
+$s->check(qr/^a02 BAD/, 'imap pipelined second noop rejected');
+$s->check(qr/^$/, 'imap pipelined max commands');
+
+# pop3
+
+$s = Test::Nginx::POP3->new();
+$s->read();
+
+$s->send('NOOP');
+$s->check(qr/^\+OK/, 'pop3 first noop');
+$s->send('NOOP');
+$s->check(qr/^-ERR/, 'pop3 second noop');
+$s->send('NOOP');
+$s->check(qr/^$/, 'pop3 max commands');
+
+$s = Test::Nginx::POP3->new();
+$s->read();
+
+$s->send('NOOP' . CRLF . 'NOOP' . CRLF . 'NOOP');
+$s->check(qr/^\+OK/, 'pop3 pipelined first noop');
+$s->check(qr/^-ERR/, 'pop3 pipelined second noop rejected');
+$s->check(qr/^$/, 'pop3 pipelined max commands');
+
+# smtp
+
+$s = Test::Nginx::SMTP->new();
+$s->read();
+
+$s->send('RSET');
+$s->check(qr/^2.. /, 'smtp first rset');
+$s->send('RSET');
+$s->check(qr/^5.. /, 'smtp second rset rejected');
+$s->send('RSET');
+$s->check(qr/^$/, 'smtp max commands');
+
+$s = Test::Nginx::SMTP->new();
+$s->read();
+
+$s->send('RSET' . CRLF . 'RSET' . CRLF . 'RSET');
+$s->check(qr/^2.. /, 'smtp pipelined first rset');
+$s->check(qr/^5.. /, 'smtp pipelined second rset rejected');
+$s->check(qr/^$/, 'smtp pipelined max commands');
+
+###############################################################################


From mdounin at mdounin.ru  Sat Mar 30 05:08:35 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 08:08:35 +0300
Subject: [nginx-tests] Tests: tests for request body chunked extensions a...
Message-ID: <hg.5cf0e07d63a1.1711775315.-1911883231343982053@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx-tests/rev/5cf0e07d63a1
branches:  
changeset: 1954:5cf0e07d63a1
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 07:48:42 2024 +0300
description:
Tests: tests for request body chunked extensions and trailers.

diffstat:

 body_chunked.t |  65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 64 insertions(+), 1 deletions(-)

diffs (96 lines):

diff --git a/body_chunked.t b/body_chunked.t
--- a/body_chunked.t
+++ b/body_chunked.t
@@ -22,7 +22,7 @@ use Test::Nginx;
 select STDERR; $| = 1;
 select STDOUT; $| = 1;
 
-my $t = Test::Nginx->new()->has(qw/http proxy rewrite/)->plan(18);
+my $t = Test::Nginx->new()->has(qw/http proxy rewrite/)->plan(24);
 
 $t->write_file_expand('nginx.conf', <<'EOF');
 
@@ -68,6 +68,7 @@ http {
         }
         location /large {
             client_max_body_size 1k;
+            client_body_buffer_size 2k;
             proxy_pass http://127.0.0.1:8081;
         }
         location /discard {
@@ -119,6 +120,8 @@ like(http_get_body('/single', '012345678
 	qr/X-Body: (0123456789){128}\x0d?$/ms, 'body in single buffer');
 
 like(http_get_body('/large', '0123456789' x 128), qr/ 413 /, 'body too large');
+like(http_get_body('/large', 'X' x 1024), qr/ 200 /, 'body exact limit');
+like(http_get_body('/large', 'X' x 1025), qr/ 413 /, 'body just above limit');
 
 # pipelined requests
 
@@ -162,6 +165,66 @@ like(
 	qr/400 Bad/, 'runaway chunk discard'
 );
 
+# chunk extensions and trailers
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. ('1; foo' . CRLF . 'X' . CRLF) x 16
+		. '0' . CRLF . CRLF
+	),
+	qr/ 200 /, 'chunk extensions'
+);
+
+TODO: {
+local $TODO = 'not yet' unless $t->has_version('1.25.5');
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. ('1; foo' . CRLF . 'X' . CRLF) x 512
+		. '0' . CRLF . CRLF
+	),
+	qr/ 413 /, 'too many chunk extensions'
+);
+
+}
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. '1' . CRLF . 'X' . CRLF
+		. '0' . CRLF . ('X-Trailer: foo' . CRLF) x 16 . CRLF
+	),
+	qr/ 200 /, 'trailers'
+);
+
+TODO: {
+local $TODO = 'not yet' unless $t->has_version('1.25.5');
+
+like(
+	http(
+		'GET /large HTTP/1.1' . CRLF
+		. 'Host: localhost' . CRLF
+		. 'Connection: close' . CRLF
+		. 'Transfer-Encoding: chunked' . CRLF . CRLF
+		. '1' . CRLF . 'X' . CRLF
+		. '0' . CRLF . ('X-Trailer: foo' . CRLF) x 512 . CRLF
+	),
+	qr/ 413 /, 'too many trailers'
+);
+
+}
+
 # proxy_next_upstream
 
 like(http_get_body('/next', '0123456789'),


From mdounin at mdounin.ru  Sat Mar 30 05:08:35 2024
From: mdounin at mdounin.ru (=?iso-8859-1?q?Maxim_Dounin?=)
Date: Sat, 30 Mar 2024 08:08:35 +0300
Subject: [nginx-tests] Tests: basic request parsing tests.
Message-ID: <hg.0b1a671c20c1.1711775315.-1911883231343982053@ns3.mdounin.ru>

details:   http://freenginx.org/hg/nginx-tests/rev/0b1a671c20c1
branches:  
changeset: 1955:0b1a671c20c1
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Mar 30 07:49:05 2024 +0300
description:
Tests: basic request parsing tests.

diffstat:

 http_request.t |  190 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 190 insertions(+), 0 deletions(-)

diffs (195 lines):

diff --git a/http_request.t b/http_request.t
new file mode 100644
--- /dev/null
+++ b/http_request.t
@@ -0,0 +1,190 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Tests for basic HTTP request parsing.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+
+use Socket qw/ CRLF CR LF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()->has(qw/http rewrite/)->plan(40)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+    %%TEST_GLOBALS_HTTP%%
+
+    server {
+        listen 127.0.0.1:8080;
+        return 200 ok\n;
+    }
+}
+
+EOF
+
+$t->run();
+
+###############################################################################
+
+# some basic HTTP/0.9, HTTP/1.0, and HTTP/1.1 requests
+
+like(http(
+	"GET /" . CRLF
+), qr/^ok/s, 'http/0.9 request');
+
+like(http(
+	"GET / HTTP/1.0" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.0 request');
+
+like(http(
+	"GET / HTTP/1.0" . CRLF .
+	"Host: foo" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.0 request with host');
+
+like(http(
+	"GET / HTTP/1.1" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.1 request');
+
+like(http(
+	"GET / HTTP/1.1" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 400 /, 'http/1.1 request rejected without host');
+
+like(http(
+	"GET http://foo/ HTTP/1.1" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'http/1.1 request absolute form');
+
+# ensure an empty line is ignored before the request
+
+like(http(CRLF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'empty line ignored');
+like(http(LF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'empty line with just LF ignored');
+
+TODO: {
+local $TODO = 'not yet' unless $t->has_version('1.25.5');
+
+like(http(CR . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'empty line with just CR rejected');
+like(http(CRLF . CRLF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple empty lines rejected');
+like(http(LF . LF . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple LFs rejected');
+like(http(CR . CR . "GET / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'multiple CRs rejected');
+
+}
+
+# method
+
+like(http("FOO / HTTP/1.0" . CRLF . CRLF), qr/ 200 /, 'method');
+like(http("FOO-BAR / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'method with dash');
+like(http("FOO_BAR / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'method with underscore');
+like(http("FOO.BAR / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'method with dot rejected');
+like(http("get / HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'method in lowercase rejected');
+
+# URI
+
+like(http("GET /foo12.bar HTTP/1.0" . CRLF . CRLF), qr/ 200 /, 'uri');
+like(http("GET /control\x0d HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with CR');
+like(http("GET /control\x01 HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with control');
+like(http("GET /control\t HTTP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'uri with tab');
+
+# version
+
+like(http(
+	"GET / HTTP/1.2" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'version 1.2');
+
+like(http(
+	"GET / HTTP/1.99" . CRLF .
+	"Host: foo" . CRLF .
+	"Connection: close" . CRLF .
+	CRLF
+), qr/ 200 /, 'version 1.99');
+
+like(http("GET / HTTP/1.000" . CRLF . CRLF), qr/ 200 /,
+	'version leading zeros');
+like(http("GET / HTTP/2.0" . CRLF . CRLF), qr/ 505 /,
+	'version too high rejected');
+like(http("GET / HTTP/1.x" . CRLF . CRLF), qr/ 400 /,
+	'version non-numeric rejected');
+like(http("GET / HTTP/1.100" . CRLF . CRLF), qr/ 400 /,
+	'version too high minor rejected');
+
+like(http("GET / http/1.0" . CRLF . CRLF), qr/ 400 /,
+	'lowercase protocol rejected');
+
+# spaces in request line
+
+like(http("GET / HTTP/1.0  " . CRLF . CRLF), qr/ 200 /,
+	'spaces after version');
+like(http("GET /   HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'spaces after uri');
+like(http("GET   / HTTP/1.0" . CRLF . CRLF), qr/ 200 /,
+	'spaces before uri');
+
+like(http("GET / HTTP/ 1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces before version rejected');
+like(http("GET / HTTP /1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces after protocol rejected');
+like(http("GET / HT TP/1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces within protocol rejected');
+like(http(" GET / HTTP/ 1.0" . CRLF . CRLF), qr/ 400 /,
+	'spaces before method rejected');
+
+# headers
+
+like(http("GET / HTTP/1.0" . CRLF . "Foo: bar" . CRLF . CRLF), qr/ 200 /,
+	'header');
+like(http("GET / HTTP/1.0" . CRLF . "Foo : bar" . CRLF . CRLF), qr/ 400 /,
+	'header with space rejected');
+like(http("GET / HTTP/1.0" . CRLF . " Foo: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with leading space rejected');
+like(http("GET / HTTP/1.0" . CRLF . "Foo\x01: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with control rejected');
+like(http("GET / HTTP/1.0" . CRLF . "Foo\t: bar" . CRLF . CRLF), qr/ 400 /,
+	'header with tab rejected');
+
+###############################################################################


From mdounin at mdounin.ru  Sat Mar 30 21:12:36 2024
From: mdounin at mdounin.ru (Maxim Dounin)
Date: Sun, 31 Mar 2024 00:12:36 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <bd554a2d48fad3f2f95bebdfc3dbbeb7@elven.pw>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru>
 <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru>
 <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru>
 <a6866930770620d19f919657974efc96@elven.pw>
 <ZgTT7R5VkW6pkwmU@mdounin.ru>
 <bd554a2d48fad3f2f95bebdfc3dbbeb7@elven.pw>
Message-ID: <ZgiARBmE2A7oCnOO@mdounin.ru>

Hello!

On Fri, Mar 29, 2024 at 08:25:18PM +0300, Lafiel wrote:

> Hello!
> 
> Maxim Dounin ?????(?) 2024-03-28 05:20:
> > It's in TODO list.
> 
> Then should take a closer look at Forgejo. It is developing a new
> ForgeFed protocol for collaborative development:
> https://forgefed.org

Interesting, thanks.

> > Well, number of extensions mapped to application/octet-stream is
> > certainly shouldn't be a deciding factor - this is essentially a
> > catch-all type, suitable for most files.
> 
> The configuration file `conf/nginx.conf` contains parameter
> `default_type application/octet-stream;`. It is entirely possible
> to change it to default instead of `text/plain` and remove remaining
> `application/octet-stream` types from the mime.types file.
> Small optimization :)

The mime.types file does not assume any default type, and rather 
specifies usable types for well-known extensions.  It is expected 
to work with all possible default types set - including text/plain 
(which is the default), application/octet-stream (which is in the 
example configuration and believed to be better suited for 
configurations with at least some basic mime.types available), and 
even no default type at all (in case we'll ever decide to follow 
Apache and switch to no default type, see 
https://bz.apache.org/bugzilla/show_bug.cgi?id=13986).

That is, removing application/octet-stream extensions from the 
mime.types file does not look like a valid option.

> > Further, application/vnd.microsoft.portable-executable type looks
> > questionable.  In particular, Microsoft's own IIS uses
> > application/x-ms-download for "dll" files, and
> > application/octet-stream for "exe" files.  Apache uses
> > application/x-msdownload for both.
> > 
> > Based on this data, I would rather refrain from changes.  Thanks
> > for trying.
> 
> I checked that they use `application/octet-stream` type on their servers:
> $ curl --head https://download.microsoft.com/download/1/7/1/1718CCC4-6315-4D8E-9543-8E28A4E18C4C/dxwebsetup.exe
> 2>1 | grep 'Content-Type:'
> Content-Type: application/octet-stream

Just for the record, I've checked IIS mappings as available in 
applicationHost.config on hosts with IIS installed.  Not sure if 
there is a current mapping officially available online, but at 
least some not very old copy can be found here:

https://github.com/aspnet/AspLabs/blob/main/src/dotnet-iis/src/dotnet-iis/Resources/applicationHost.config#L370

> > It is not clear how Boulder handling of the Accept request header
> > is relevant here.
> > 
> > Also, link [3] contains at least two comments which are directly
> > relevant and contradicts to what the patch suggests, notably:
> > 
> > https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/2
> > 
> > "You are also asking about file extensions being .cer, .crt, .pem,
> > but there really isn't a standardized meaning for those extensions
> > or their contents. Certbot uses .pem for files that are for a
> > single certificate, a chain of certificates, or a private key, and
> > much other software integrating with ACME follows that convention
> > as well. Windows tends to use .cer for single certificates and
> > .crt for certificates intended to be used as trust anchors, but
> > doesn't care if they're DER-encoded or PEM-encoded, they just have
> > different default behavior when double-clicking on them (as .crt
> > will default to asking if you want to add it to your trust store
> > and .cer will default to just showing you, at least when I last
> > tried them)."
> > 
> > And
> > https://community.letsencrypt.org/t/formats-of-certificates-issued-by-lets-encrypt/194990/4,
> > which suggests mapping from extensions to MIME types.  In
> > particular, it uses application/x-pem-file for pem files.
> > 
> > Similarly, letsencrypt.org is using application/x-pem-file for pem
> > files, notably for their own keys.
> 
> I wanted to give as an example that changing type to another should not
> affect the work.

Well, Boulder simply ignores the Accept request header, that is, 
it ignores client preference for a particular type, nothing more.  
It provides no information if changing the response type will 
affect various clients or not.

> > Unless there are requests, I would rather refrain from adding
> > it.  Also, if at all, this probably should be added along with
> > "crl" from the same RFC.
> > 
> > Dropped this for now, thanks.
> 
> There is no .crl extension in httparchive statistics, so I didn?t add it.
> I can try updating the commit :)

I think we can consider this at some point later, if/when there 
will be more information available.

-- 
Maxim Dounin
http://mdounin.ru/


From lafiel at elven.pw  Sun Mar 31 12:34:24 2024
From: lafiel at elven.pw (Lafiel)
Date: Sun, 31 Mar 2024 15:34:24 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZgiARBmE2A7oCnOO@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru> <a6866930770620d19f919657974efc96@elven.pw>
 <ZgTT7R5VkW6pkwmU@mdounin.ru> <bd554a2d48fad3f2f95bebdfc3dbbeb7@elven.pw>
 <ZgiARBmE2A7oCnOO@mdounin.ru>
Message-ID: <d33b4114a01b44c6875c77aa059c72ee@elven.pw>

Hello!

Maxim 
Dounin 
?????(?) 
2024-03-31 
00:12:
> Just for the record, I've checked IIS mappings as available 
> in
> applicationHost.config on hosts with IIS installed.  Not sure 
> if
> there is a current mapping officially available online, but 
> at
> least some not very old copy can be found 
> here:
> 
> https://github.com/aspnet/AspLabs/blob/main/src/dotnet-iis/src/dotnet-iis/Resources/applicationHost.config#L370

As 
in 
Apache, 
a 
large 
of 
extensions 
in 
mime 
types 
are 
specified
here 
for 
Excel 
and 
PowerPoint 
:)


> Well, Boulder simply ignores the Accept request header, that 
> is,
> it ignores client preference for a particular type, nothing 
> more.
> It provides no information if changing the response type 
> will
> affect various clients or 
> not.

Tried 
updating 
comit 
and 
added 
more 
information 
with 
examples.


> I think we can consider this at some point later, if/when 
> there
> will be more information 
> available.

Tried 
updating 
comit.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mime_types_02.patch
Type: text/x-diff
Size: 6415 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment.patch>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment.sig>

From lafiel at elven.pw  Sun Mar 31 12:34:28 2024
From: lafiel at elven.pw (Lafiel)
Date: Sun, 31 Mar 2024 15:34:28 +0300
Subject: [nginx] Update mime-types
In-Reply-To: <ZgiARBmE2A7oCnOO@mdounin.ru>
References: <6fd7b4aae9f284157bcd51b2eb936b82@elven.pw>
 <ZdlN9mjsSgsQvaw8@mdounin.ru> <21d6fa47ad3cc9ef2a2fc54f85a7349f@elven.pw>
 <ZdvIZ3tTgUF5K5hJ@mdounin.ru> <4943345f2efed54da9bddc75355fe1fa@elven.pw>
 <ZgJBG8o4Y4y5ManS@mdounin.ru> <a6866930770620d19f919657974efc96@elven.pw>
 <ZgTT7R5VkW6pkwmU@mdounin.ru> <bd554a2d48fad3f2f95bebdfc3dbbeb7@elven.pw>
 <ZgiARBmE2A7oCnOO@mdounin.ru>
Message-ID: <d33b4114a01b44c6875c77aa059c72ee@elven.pw>

Hello!

Maxim 
Dounin 
?????(?) 
2024-03-31 
00:12:
> Just for the record, I've checked IIS mappings as available 
> in
> applicationHost.config on hosts with IIS installed.  Not sure 
> if
> there is a current mapping officially available online, but 
> at
> least some not very old copy can be found 
> here:
> 
> https://github.com/aspnet/AspLabs/blob/main/src/dotnet-iis/src/dotnet-iis/Resources/applicationHost.config#L370

As 
in 
Apache, 
a 
large 
of 
extensions 
in 
mime 
types 
are 
specified
here 
for 
Excel 
and 
PowerPoint 
:)


> Well, Boulder simply ignores the Accept request header, that 
> is,
> it ignores client preference for a particular type, nothing 
> more.
> It provides no information if changing the response type 
> will
> affect various clients or 
> not.

Tried 
updating 
comit 
and 
added 
more 
information 
with 
examples.


> I think we can consider this at some point later, if/when 
> there
> will be more information 
> available.

Tried 
updating 
comit.


-- 
Best 
regards,
Lafiel
mailto:lafiel at elven.pw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mime_types_02.patch
Type: text/x-diff
Size: 6415 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment-0001.patch>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0xFAB0C3D2.asc
Type: application/pgp-keys
Size: 1461 bytes
Desc: not available
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment-0001.asc>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 228 bytes
Desc: OpenPGP digital signature
URL: <http://freenginx.org/pipermail/nginx-devel/attachments/20240331/ed2addbe/attachment-0001.sig>

From mdounin at mdounin.ru  Sun Mar 31 21:38:43 2024
From: mdounin at mdounin.ru (=?utf-8?q?Maxim_Dounin?=)
Date: Mon, 01 Apr 2024 00:38:43 +0300
Subject: [PATCH] Documented max_commands in mail module
Message-ID: <b9fa43195536cb6cd574.1711921123@vm-bsd.mdounin.ru>

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1711920939 -10800
#      Mon Apr 01 00:35:39 2024 +0300
# Node ID b9fa43195536cb6cd5744f929b89f44f79b338fb
# Parent  5a0725cb366bc4c98c7e2e4af2f1d32d6a5d85a6
Documented max_commands in mail module.

diff --git a/xml/en/docs/mail/ngx_mail_core_module.xml b/xml/en/docs/mail/ngx_mail_core_module.xml
--- a/xml/en/docs/mail/ngx_mail_core_module.xml
+++ b/xml/en/docs/mail/ngx_mail_core_module.xml
@@ -10,7 +10,7 @@
 <module name="Module ngx_mail_core_module"
         link="/en/docs/mail/ngx_mail_core_module.html"
         lang="en"
-        rev="22">
+        rev="23">
 
 <section id="summary">
 
@@ -247,6 +247,22 @@ are specified.
 </directive>
 
 
+<directive name="max_commands">
+<syntax><value>number</value></syntax>
+<default>1000</default>
+<context>mail</context>
+<context>server</context>
+<appeared-in>1.25.5</appeared-in>
+
+<para>
+Sets the maximum number of commands allowed during authentication.
+If the limit is reached and proxying to the backend is not started yet,
+the connection is closed.
+</para>
+
+</directive>
+
+
 <directive name="max_errors">
 <syntax><value>number</value></syntax>
 <default>5</default>
diff --git a/xml/ru/docs/mail/ngx_mail_core_module.xml b/xml/ru/docs/mail/ngx_mail_core_module.xml
--- a/xml/ru/docs/mail/ngx_mail_core_module.xml
+++ b/xml/ru/docs/mail/ngx_mail_core_module.xml
@@ -10,7 +10,7 @@
 <module name="?????? ngx_mail_core_module"
         link="/ru/docs/mail/ngx_mail_core_module.html"
         lang="ru"
-        rev="22">
+        rev="23">
 
 <section id="summary">
 
@@ -251,6 +251,22 @@ FreeBSD?9.0-STABLE)
 </directive>
 
 
+<directive name="max_commands">
+<syntax><value>?????</value></syntax>
+<default>1000</default>
+<context>mail</context>
+<context>server</context>
+<appeared-in>1.25.5</appeared-in>
+
+<para>
+?????? ???????????? ?????????? ???????, ??????????? ? ???????? ??????????????.
+???? ??????????? ??????????, ? ????????????? ?? ?????? ?? ????????,
+?????????? ???????????.
+</para>
+
+</directive>
+
+
 <directive name="max_errors">
 <syntax><value>?????</value></syntax>
 <default>5</default>