Mercurial > hg > nginx
view src/event/ngx_event_udp.c @ 8950:fb811b6c76ee quic
QUIC: refactored buffer allocation, spliting and freeing.
Previously, buffer lists was used to track used buffers. Now reference
counter is used instead. The new implementation is simpler and faster with
many buffer clones.
author | Roman Arutyunyan <arut@nginx.com> |
---|---|
date | Fri, 24 Dec 2021 18:39:22 +0300 |
parents | e7a2d3914877 |
children | ce6d9cf0f567 |
line wrap: on
line source
/* * Copyright (C) Roman Arutyunyan * Copyright (C) Nginx, Inc. */ #include <ngx_config.h> #include <ngx_core.h> #include <ngx_event.h> #if !(NGX_WIN32) static void ngx_close_accepted_udp_connection(ngx_connection_t *c); static ssize_t ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf, size_t size); static ngx_int_t ngx_create_udp_connection(ngx_connection_t *c); static ngx_connection_t *ngx_lookup_udp_connection(ngx_listening_t *ls, ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen); void ngx_event_recvmsg(ngx_event_t *ev) { size_t len; ssize_t n; ngx_str_t key; ngx_buf_t buf; ngx_log_t *log; ngx_err_t err; socklen_t local_socklen; ngx_event_t *rev, *wev; struct iovec iov[1]; struct msghdr msg; ngx_sockaddr_t sa, lsa; ngx_udp_dgram_t dgram; struct sockaddr *local_sockaddr; ngx_listening_t *ls; ngx_event_conf_t *ecf; ngx_connection_t *c, *lc; static u_char buffer[65535]; #if (NGX_HAVE_ADDRINFO_CMSG) u_char msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))]; #endif if (ev->timedout) { if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { return; } ev->timedout = 0; } ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { ev->available = ecf->multi_accept; } lc = ev->data; ls = lc->listening; ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "recvmsg on %V, ready: %d", &ls->addr_text, ev->available); do { ngx_memzero(&msg, sizeof(struct msghdr)); iov[0].iov_base = (void *) buffer; iov[0].iov_len = sizeof(buffer); msg.msg_name = &sa; msg.msg_namelen = sizeof(ngx_sockaddr_t); msg.msg_iov = iov; msg.msg_iovlen = 1; #if (NGX_HAVE_ADDRINFO_CMSG) if (ls->wildcard) { msg.msg_control = &msg_control; msg.msg_controllen = sizeof(msg_control); ngx_memzero(&msg_control, sizeof(msg_control)); } #endif n = recvmsg(lc->fd, &msg, 0); if (n == -1) { err = ngx_socket_errno; if (err == NGX_EAGAIN) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "recvmsg() not ready"); return; } ngx_log_error(NGX_LOG_ALERT, ev->log, err, "recvmsg() failed"); return; } #if (NGX_HAVE_ADDRINFO_CMSG) if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) { ngx_log_error(NGX_LOG_ALERT, ev->log, 0, "recvmsg() truncated data"); continue; } #endif dgram.sockaddr = msg.msg_name; dgram.socklen = msg.msg_namelen; if (dgram.socklen > (socklen_t) sizeof(ngx_sockaddr_t)) { dgram.socklen = sizeof(ngx_sockaddr_t); } if (dgram.socklen == 0) { /* * on Linux recvmsg() returns zero msg_namelen * when receiving packets from unbound AF_UNIX sockets */ dgram.socklen = sizeof(struct sockaddr); ngx_memzero(&sa, sizeof(struct sockaddr)); sa.sockaddr.sa_family = ls->sockaddr->sa_family; } local_sockaddr = ls->sockaddr; local_socklen = ls->socklen; #if (NGX_HAVE_ADDRINFO_CMSG) if (ls->wildcard) { struct cmsghdr *cmsg; ngx_memcpy(&lsa, local_sockaddr, local_socklen); local_sockaddr = &lsa.sockaddr; for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { if (ngx_get_srcaddr_cmsg(cmsg, local_sockaddr) == NGX_OK) { break; } } } #endif key.data = (u_char *) dgram.sockaddr; key.len = dgram.socklen; #if (NGX_HAVE_UNIX_DOMAIN) if (dgram.sockaddr->sa_family == AF_UNIX) { struct sockaddr_un *saun = (struct sockaddr_un *) dgram.sockaddr; if (dgram.socklen <= (socklen_t) offsetof(struct sockaddr_un, sun_path) || saun->sun_path[0] == '\0') { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0, "unbound unix socket"); key.len = 0; } } #endif #if (NGX_QUIC) if (ls->quic) { if (ngx_quic_get_packet_dcid(ev->log, buffer, n, &key) != NGX_OK) { goto next; } } #endif c = ngx_lookup_udp_connection(ls, &key, local_sockaddr, local_socklen); if (c) { #if (NGX_DEBUG) if (c->log->log_level & NGX_LOG_DEBUG_EVENT) { ngx_log_handler_pt handler; handler = c->log->handler; c->log->handler = NULL; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "recvmsg: fd:%d n:%z", c->fd, n); c->log->handler = handler; } #endif ngx_memzero(&buf, sizeof(ngx_buf_t)); buf.pos = buffer; buf.last = buffer + n; buf.start = buf.pos; buf.end = buffer + sizeof(buffer); rev = c->read; dgram.buffer = &buf; c->udp->dgram = &dgram; rev->ready = 1; rev->active = 0; rev->handler(rev); if (c->udp) { c->udp->dgram = NULL; } rev->ready = 0; rev->active = 1; goto next; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; c = ngx_get_connection(lc->fd, ev->log); if (c == NULL) { return; } c->shared = 1; c->type = SOCK_DGRAM; c->socklen = dgram.socklen; #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif c->pool = ngx_create_pool(ls->pool_size, ev->log); if (c->pool == NULL) { ngx_close_accepted_udp_connection(c); return; } len = dgram.socklen; #if (NGX_QUIC) if (ls->quic) { len = NGX_SOCKADDRLEN; } #endif c->sockaddr = ngx_palloc(c->pool, len); if (c->sockaddr == NULL) { ngx_close_accepted_udp_connection(c); return; } ngx_memcpy(c->sockaddr, dgram.sockaddr, dgram.socklen); log = ngx_palloc(c->pool, sizeof(ngx_log_t)); if (log == NULL) { ngx_close_accepted_udp_connection(c); return; } *log = ls->log; c->recv = ngx_udp_shared_recv; c->send = ngx_udp_send; c->send_chain = ngx_udp_send_chain; c->log = log; c->pool->log = log; c->listening = ls; if (local_sockaddr == &lsa.sockaddr) { local_sockaddr = ngx_palloc(c->pool, local_socklen); if (local_sockaddr == NULL) { ngx_close_accepted_udp_connection(c); return; } ngx_memcpy(local_sockaddr, &lsa, local_socklen); } c->local_sockaddr = local_sockaddr; c->local_socklen = local_socklen; c->buffer = ngx_create_temp_buf(c->pool, n); if (c->buffer == NULL) { ngx_close_accepted_udp_connection(c); return; } c->buffer->last = ngx_cpymem(c->buffer->last, buffer, n); rev = c->read; wev = c->write; rev->active = 1; wev->ready = 1; rev->log = log; wev->log = log; /* * TODO: MT: - ngx_atomic_fetch_add() * or protection by critical section or light mutex * * TODO: MP: - allocated in a shared memory * - ngx_atomic_fetch_add() * or protection by critical section or light mutex */ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); c->start_time = ngx_current_msec; #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_handled, 1); #endif if (ls->addr_ntop) { c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len); if (c->addr_text.data == NULL) { ngx_close_accepted_udp_connection(c); return; } c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen, c->addr_text.data, ls->addr_text_max_len, 0); if (c->addr_text.len == 0) { ngx_close_accepted_udp_connection(c); return; } } #if (NGX_DEBUG) { ngx_str_t addr; u_char text[NGX_SOCKADDR_STRLEN]; ngx_debug_accepted_connection(ecf, c); if (log->log_level & NGX_LOG_DEBUG_EVENT) { addr.data = text; addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text, NGX_SOCKADDR_STRLEN, 1); ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA recvmsg: %V fd:%d n:%z", c->number, &addr, c->fd, n); } } #endif if (ngx_create_udp_connection(c) != NGX_OK) { ngx_close_accepted_udp_connection(c); return; } log->data = NULL; log->handler = NULL; ls->handler(c); next: if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available -= n; } } while (ev->available); } static void ngx_close_accepted_udp_connection(ngx_connection_t *c) { ngx_free_connection(c); c->fd = (ngx_socket_t) -1; if (c->pool) { ngx_destroy_pool(c->pool); } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, -1); #endif } static ssize_t ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf, size_t size) { ssize_t n; ngx_buf_t *b; if (c->udp == NULL || c->udp->dgram == NULL) { return NGX_AGAIN; } b = c->udp->dgram->buffer; n = ngx_min(b->last - b->pos, (ssize_t) size); ngx_memcpy(buf, b->pos, n); c->udp->dgram = NULL; c->read->ready = 0; c->read->active = 1; return n; } void ngx_udp_rbtree_insert_value(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) { ngx_int_t rc; ngx_connection_t *c, *ct; ngx_rbtree_node_t **p; ngx_udp_connection_t *udp, *udpt; for ( ;; ) { if (node->key < temp->key) { p = &temp->left; } else if (node->key > temp->key) { p = &temp->right; } else { /* node->key == temp->key */ udp = (ngx_udp_connection_t *) node; c = udp->connection; udpt = (ngx_udp_connection_t *) temp; ct = udpt->connection; rc = ngx_memn2cmp(udp->key.data, udpt->key.data, udp->key.len, udpt->key.len); if (rc == 0 && c->listening->wildcard) { rc = ngx_cmp_sockaddr(c->local_sockaddr, c->local_socklen, ct->local_sockaddr, ct->local_socklen, 1); } p = (rc < 0) ? &temp->left : &temp->right; } if (*p == sentinel) { break; } temp = *p; } *p = node; node->parent = temp; node->left = sentinel; node->right = sentinel; ngx_rbt_red(node); } static ngx_int_t ngx_create_udp_connection(ngx_connection_t *c) { ngx_str_t key; ngx_pool_cleanup_t *cln; ngx_udp_connection_t *udp; #if (NGX_QUIC) if (c->listening->quic) { return NGX_OK; } #endif if (c->udp) { return NGX_OK; } udp = ngx_pcalloc(c->pool, sizeof(ngx_udp_connection_t)); if (udp == NULL) { return NGX_ERROR; } cln = ngx_pool_cleanup_add(c->pool, 0); if (cln == NULL) { return NGX_ERROR; } cln->data = c; cln->handler = ngx_delete_udp_connection; key.data = (u_char *) c->sockaddr; key.len = c->socklen; ngx_insert_udp_connection(c, udp, &key); c->udp = udp; return NGX_OK; } void ngx_insert_udp_connection(ngx_connection_t *c, ngx_udp_connection_t *udp, ngx_str_t *key) { uint32_t hash; ngx_crc32_init(hash); ngx_crc32_update(&hash, key->data, key->len); if (c->listening->wildcard) { ngx_crc32_update(&hash, (u_char *) c->local_sockaddr, c->local_socklen); } ngx_crc32_final(hash); udp->connection = c; udp->key = *key; udp->node.key = hash; ngx_rbtree_insert(&c->listening->rbtree, &udp->node); } void ngx_delete_udp_connection(void *data) { ngx_connection_t *c = data; if (c->udp == NULL) { return; } ngx_rbtree_delete(&c->listening->rbtree, &c->udp->node); c->udp = NULL; } static ngx_connection_t * ngx_lookup_udp_connection(ngx_listening_t *ls, ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen) { uint32_t hash; ngx_int_t rc; ngx_connection_t *c; ngx_rbtree_node_t *node, *sentinel; ngx_udp_connection_t *udp; if (key->len == 0) { return NULL; } node = ls->rbtree.root; sentinel = ls->rbtree.sentinel; ngx_crc32_init(hash); ngx_crc32_update(&hash, key->data, key->len); if (ls->wildcard) { ngx_crc32_update(&hash, (u_char *) local_sockaddr, local_socklen); } ngx_crc32_final(hash); while (node != sentinel) { if (hash < node->key) { node = node->left; continue; } if (hash > node->key) { node = node->right; continue; } /* hash == node->key */ udp = (ngx_udp_connection_t *) node; c = udp->connection; rc = ngx_memn2cmp(key->data, udp->key.data, key->len, udp->key.len); if (rc == 0 && ls->wildcard) { rc = ngx_cmp_sockaddr(local_sockaddr, local_socklen, c->local_sockaddr, c->local_socklen, 1); } if (rc == 0) { #if (NGX_QUIC) if (ls->quic && c->udp != udp) { c->udp = udp; } #endif return c; } node = (rc < 0) ? node->left : node->right; } return NULL; } #else void ngx_delete_udp_connection(void *data) { return; } #endif