[nginx] Upstream: improved reporting of peer failures.

[Maxim Dounin]

details:   http://freenginx.org/hg/nginx/rev/cd3a3e585b29
branches:  
changeset: 9461:cd3a3e585b29
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Sat Jan 31 09:27:09 2026 +0300
description:
Upstream: improved reporting of peer failures.

Previously, peer failures were not reported to the balancer if an error
response listed in u->conf->next_upstream was received, but switching
to the next upstream server was not possible due to other reasons (for
example, when no u->peer.tries left, or due to unbuffered request body).

In particular, this resulted in upstream servers never switched off due
to 502 errors in configurations with "grpc_next_upstream http_502", as
gRPC proxying implies unbuffered request body
(https://freenginx.org/pipermail/nginx/2025-February/000150.html).

Fix is to preserve information about the failure as detected by
ngx_http_upstream_test_next(), and propagate it to the balancer when
u->peer.free() is called during upstream request finalization.

diffstat:

 src/http/ngx_http_upstream.c |  57 +++++++++++++++++++++++++++++--------------
 src/http/ngx_http_upstream.h |   2 +
 2 files changed, 40 insertions(+), 19 deletions(-)

diffs (86 lines):

diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -2584,24 +2584,43 @@ ngx_http_upstream_test_next(ngx_http_req
             continue;
         }
 
-        timeout = u->conf->next_upstream_timeout;
-
-        if (u->request_sent
-            && (r->method & (NGX_HTTP_POST|NGX_HTTP_LOCK|NGX_HTTP_PATCH)))
-        {
-            mask = un->mask | NGX_HTTP_UPSTREAM_FT_NON_IDEMPOTENT;
-
-        } else {
-            mask = un->mask;
-        }
-
-        if (u->peer.tries > 1
-            && ((u->conf->next_upstream & mask) == mask)
-            && !(u->request_sent && r->request_body_no_buffering)
-            && !(timeout && ngx_current_msec - u->peer.start_time >= timeout))
-        {
-            ngx_http_upstream_next(r, u, un->mask);
-            return NGX_OK;
+        if (u->conf->next_upstream & un->mask) {
+
+            timeout = u->conf->next_upstream_timeout;
+
+            if (u->request_sent
+                && (r->method & (NGX_HTTP_POST|NGX_HTTP_LOCK|NGX_HTTP_PATCH)))
+            {
+                mask = un->mask | NGX_HTTP_UPSTREAM_FT_NON_IDEMPOTENT;
+
+            } else {
+                mask = un->mask;
+            }
+
+            if (u->peer.tries > 1
+                && ((u->conf->next_upstream & mask) == mask)
+                && !(u->request_sent && r->request_body_no_buffering)
+                && !(timeout
+                     && ngx_current_msec - u->peer.start_time >= timeout))
+            {
+                ngx_http_upstream_next(r, u, un->mask);
+                return NGX_OK;
+            }
+
+            /*
+             * if we were expected to switch to the next server, but
+             * were not able to do so due to additional restrictions,
+             * remember that the peer failed
+             */
+
+            if (un->mask == NGX_HTTP_UPSTREAM_FT_HTTP_403
+                || un->mask == NGX_HTTP_UPSTREAM_FT_HTTP_404)
+            {
+                u->peer_state = NGX_PEER_NEXT;
+
+            } else {
+                u->peer_state = NGX_PEER_FAILED;
+            }
         }
 
 #if (NGX_HTTP_CACHE)
@@ -4623,7 +4642,7 @@ ngx_http_upstream_finalize_request(ngx_h
     u->finalize_request(r, rc);
 
     if (u->peer.free && u->peer.sockaddr) {
-        u->peer.free(&u->peer, u->peer.data, 0);
+        u->peer.free(&u->peer, u->peer.data, u->peer_state);
         u->peer.sockaddr = NULL;
     }
 
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -404,6 +404,8 @@ struct ngx_http_upstream_s {
     unsigned                         request_body_sent:1;
     unsigned                         request_body_blocked:1;
     unsigned                         header_sent:1;
+
+    unsigned                         peer_state:3;
 };