[nginx] Syslog: introduced ngx_syslog_send() error logging moder...

[Maxim Dounin]

details:   http://freenginx.org/hg/nginx/rev/1c9264603adc
branches:  
changeset: 9225:1c9264603adc
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Fri Mar 01 05:42:12 2024 +0300
description:
Syslog: introduced ngx_syslog_send() error logging moderation.

Errors when logging to syslog are now logged at most once per second.
This ensures that persistent errors won't flood other logs, and spontaneous
errors, such as ENOBUFS as observed on BSD systems when syslogd cannot cope
with load, or EAGAIN as seen in similar situation on Linux, won't further
overload logging subsystem, leading to more errors.

Further, errors now can only trigger reconnects at most once per second.
This ensures that persistent errors, which cannot be fixed with reconnects,
don't trigger too much unneeded work.

Additionally, in case of connection errors, such as when syslogd is not
running, connection attempts are only made once per second.

diffstat:

 src/core/ngx_connection.c |   7 +++++++
 src/core/ngx_connection.h |   3 ++-
 src/core/ngx_syslog.c     |  25 ++++++++++++++++++++++---
 src/core/ngx_syslog.h     |   3 +++
 4 files changed, 34 insertions(+), 4 deletions(-)

diffs (128 lines):

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1604,6 +1604,10 @@ ngx_connection_error(ngx_connection_t *c
     {
         switch (c->log_error) {
 
+        case NGX_ERROR_DEBUG:
+            level = NGX_LOG_DEBUG;
+            break;
+
         case NGX_ERROR_IGNORE_EMSGSIZE:
         case NGX_ERROR_IGNORE_EINVAL:
         case NGX_ERROR_IGNORE_ECONNRESET:
@@ -1615,6 +1619,9 @@ ngx_connection_error(ngx_connection_t *c
             level = NGX_LOG_ERR;
         }
 
+    } else if (c->log_error == NGX_ERROR_DEBUG) {
+        level = NGX_LOG_DEBUG;
+
     } else {
         level = NGX_LOG_ALERT;
     }
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -98,7 +98,8 @@ typedef enum {
     NGX_ERROR_INFO,
     NGX_ERROR_IGNORE_ECONNRESET,
     NGX_ERROR_IGNORE_EINVAL,
-    NGX_ERROR_IGNORE_EMSGSIZE
+    NGX_ERROR_IGNORE_EMSGSIZE,
+    NGX_ERROR_DEBUG
 } ngx_connection_log_error_e;
 
 
diff --git a/src/core/ngx_syslog.c b/src/core/ngx_syslog.c
--- a/src/core/ngx_syslog.c
+++ b/src/core/ngx_syslog.c
@@ -303,6 +303,10 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         }
     }
 
+    if (ngx_time() == peer->error_log_time) {
+        peer->conn.log_error = NGX_ERROR_DEBUG;
+    }
+
     if (ngx_send) {
         n = ngx_send(&peer->conn, buf, len);
 
@@ -311,20 +315,27 @@ ngx_syslog_send(ngx_syslog_peer_t *peer,
         n = ngx_os_io.send(&peer->conn, buf, len);
     }
 
-    if (n == NGX_ERROR) {
-
+    if (n == NGX_ERROR
+        && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         if (ngx_close_socket(peer->conn.fd) == -1) {
             ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                           ngx_close_socket_n " failed");
         }
 
         peer->conn.fd = (ngx_socket_t) -1;
+        peer->error_log_time = ngx_time();
 
-    } else if ((size_t) n != len) {
+    } else if ((size_t) n != len
+               && peer->conn.log_error != NGX_ERROR_DEBUG)
+    {
         ngx_log_error(NGX_LOG_CRIT, &peer->log, 0,
                       "send() incomplete");
+        peer->error_log_time = ngx_time();
     }
 
+    peer->conn.log_error = NGX_ERROR_ERR;
+
     return n;
 }
 
@@ -334,10 +345,15 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 {
     ngx_socket_t  fd;
 
+    if (ngx_time() == peer->connect_error_time) {
+        return NGX_ERROR;
+    }
+
     fd = ngx_socket(peer->server.sockaddr->sa_family, SOCK_DGRAM, 0);
     if (fd == (ngx_socket_t) -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_socket_n " failed");
+        peer->connect_error_time = ngx_time();
         return NGX_ERROR;
     }
 
@@ -355,6 +371,7 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
     peer->conn.fd = fd;
     peer->conn.log = &peer->log;
+    peer->conn.log_error = NGX_ERROR_ERR;
 
     /* UDP sockets are always ready to write */
     peer->conn.write->ready = 1;
@@ -363,6 +380,8 @@ ngx_syslog_init_peer(ngx_syslog_peer_t *
 
 failed:
 
+    peer->connect_error_time = ngx_time();
+
     if (ngx_close_socket(fd) == -1) {
         ngx_log_error(NGX_LOG_ALERT, &peer->log, ngx_socket_errno,
                       ngx_close_socket_n " failed");
diff --git a/src/core/ngx_syslog.h b/src/core/ngx_syslog.h
--- a/src/core/ngx_syslog.h
+++ b/src/core/ngx_syslog.h
@@ -21,6 +21,9 @@ typedef struct {
     ngx_log_t          log;
     ngx_log_t         *logp;
 
+    time_t             error_log_time;
+    time_t             connect_error_time;
+
     unsigned           busy:1;
     unsigned           nohostname:1;
 } ngx_syslog_peer_t;